From e63a886f040a0586dce23d022d72120e30feea77 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Sat, 8 Dec 2018 12:09:26 -0500 Subject: [PATCH] Performance tweaks (#127) --- .data/Dockerfile | 2 +- cmd/analyze.go | 9 ++++- cmd/build.go | 9 ++++- cmd/root.go | 4 ++ filetree/cache.go | 74 +++++++++++++++++++++++++++++++++++++ filetree/data.go | 70 +++++++++++++++-------------------- filetree/data_test.go | 2 +- filetree/efficiency.go | 19 ++-------- filetree/efficiency_test.go | 9 ++--- filetree/node.go | 38 ++++++------------- filetree/node_test.go | 29 +++++++-------- filetree/tree.go | 66 +++++++++++++++++++++------------ filetree/tree_test.go | 4 +- filetree/types.go | 65 ++++++++++++++++++++++++++++++++ image/docker_image.go | 14 +++---- ui/filetreeview.go | 14 +++---- ui/ui.go | 14 +++++-- 17 files changed, 290 insertions(+), 152 deletions(-) create mode 100644 filetree/cache.go create mode 100644 filetree/types.go diff --git a/.data/Dockerfile b/.data/Dockerfile index fc82c55..1ed949c 100644 --- a/.data/Dockerfile +++ b/.data/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest ADD README.md /somefile.txt -RUN mkdir /root/example +RUN mkdir -p /root/example/really/nested RUN cp /somefile.txt /root/example/somefile1.txt RUN chmod 444 /root/example/somefile1.txt RUN cp /somefile.txt /root/example/somefile2.txt diff --git a/cmd/analyze.go b/cmd/analyze.go index ab28979..184ef90 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -2,9 +2,9 @@ package cmd import ( "fmt" - "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/wagoodman/dive/filetree" "github.com/wagoodman/dive/image" "github.com/wagoodman/dive/ui" "github.com/wagoodman/dive/utils" @@ -33,8 +33,13 @@ func doAnalyzeCmd(cmd *cobra.Command, args []string) { utils.Exit(1) } color.New(color.Bold).Println("Analyzing Image") + result := fetchAndAnalyze(userImage) - ui.Run(fetchAndAnalyze(userImage)) + fmt.Println(" Building cache...") + cache := filetree.NewFileTreeCache(result.RefTrees) + cache.Build() + + ui.Run(result, cache) } func fetchAndAnalyze(imageID string) *image.AnalysisResult { diff --git a/cmd/build.go b/cmd/build.go index 680fc5f..315e574 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -1,9 +1,11 @@ package cmd import ( + "fmt" "github.com/fatih/color" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/wagoodman/dive/filetree" "github.com/wagoodman/dive/ui" "github.com/wagoodman/dive/utils" "io/ioutil" @@ -46,6 +48,11 @@ func doBuildCmd(cmd *cobra.Command, args []string) { } color.New(color.Bold).Println("Analyzing Image") + result := fetchAndAnalyze(string(imageId)) - ui.Run(fetchAndAnalyze(string(imageId))) + fmt.Println(" Building cache...") + cache := filetree.NewFileTreeCache(result.RefTrees) + cache.Build() + + ui.Run(result, cache) } diff --git a/cmd/root.go b/cmd/root.go index 1ec4d3f..aa97fea 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/wagoodman/dive/filetree" "github.com/wagoodman/dive/utils" "io/ioutil" "os" @@ -110,6 +111,9 @@ func initConfig() { if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } + + // set global defaults (for performance) + filetree.GlobalFileTreeCollapse = viper.GetBool("filetree.collapse-dir") } // initLogging sets up the logging object with a formatter and location diff --git a/filetree/cache.go b/filetree/cache.go new file mode 100644 index 0000000..0b19fc6 --- /dev/null +++ b/filetree/cache.go @@ -0,0 +1,74 @@ +package filetree + +type TreeCacheKey struct { + bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int +} + +type TreeCache struct { + refTrees []*FileTree + cache map[TreeCacheKey]*FileTree +} + +func (cache *TreeCache) Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int) *FileTree { + key := TreeCacheKey{bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop} + if value, exists := cache.cache[key]; exists { + return value + } else { + + } + value := cache.buildTree(key) + cache.cache[key] = value + return value +} + +func (cache *TreeCache) buildTree(key TreeCacheKey) *FileTree { + newTree := StackTreeRange(cache.refTrees, key.bottomTreeStart, key.bottomTreeStop) + + for idx := key.topTreeStart; idx <= key.topTreeStop; idx++ { + newTree.Compare(cache.refTrees[idx]) + } + return newTree +} + +func (cache *TreeCache) Build() { + var bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int + + // case 1: layer compare (top tree SIZE is fixed (BUT floats forward), Bottom tree SIZE changes) + for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ { + bottomTreeStart = 0 + topTreeStop = selectIdx + + if selectIdx == 0 { + bottomTreeStop = selectIdx + topTreeStart = selectIdx + } else { + bottomTreeStop = selectIdx - 1 + topTreeStart = selectIdx + } + + cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop) + } + + // case 2: aggregated compare (bottom tree is ENTIRELY fixed, top tree SIZE changes) + for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ { + bottomTreeStart = 0 + topTreeStop = selectIdx + if selectIdx == 0 { + bottomTreeStop = selectIdx + topTreeStart = selectIdx + } else { + bottomTreeStop = 0 + topTreeStart = 1 + } + + cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop) + } +} + +func NewFileTreeCache(refTrees []*FileTree) TreeCache { + + return TreeCache{ + refTrees: refTrees, + cache: make(map[TreeCacheKey]*FileTree), + } +} diff --git a/filetree/data.go b/filetree/data.go index 47fb89d..379454e 100644 --- a/filetree/data.go +++ b/filetree/data.go @@ -7,7 +7,6 @@ import ( "github.com/cespare/xxhash" "github.com/sirupsen/logrus" - "github.com/spf13/viper" ) const ( @@ -17,29 +16,7 @@ const ( Removed ) -// NodeData is the payload for a FileNode -type NodeData struct { - ViewInfo ViewInfo - FileInfo FileInfo - DiffType DiffType -} - -// ViewInfo contains UI specific detail for a specific FileNode -type ViewInfo struct { - Collapsed bool - Hidden bool -} - -// FileInfo contains tar metadata for a specific FileNode -type FileInfo struct { - Path string - TypeFlag byte - hash uint64 - TarHeader tar.Header -} - -// DiffType defines the comparison result between two FileNodes -type DiffType int +var GlobalFileTreeCollapse bool // NewNodeData creates an empty NodeData struct for a FileNode func NewNodeData() *NodeData { @@ -62,7 +39,7 @@ func (data *NodeData) Copy() *NodeData { // NewViewInfo creates a default ViewInfo func NewViewInfo() (view *ViewInfo) { return &ViewInfo{ - Collapsed: viper.GetBool("filetree.collapse-dir"), + Collapsed: GlobalFileTreeCollapse, Hidden: false, } } @@ -74,12 +51,10 @@ func (view *ViewInfo) Copy() (newView *ViewInfo) { return newView } -var chuckSize = 2 * 1024 * 1024 - func getHashFromReader(reader io.Reader) uint64 { h := xxhash.New() - buf := make([]byte, chuckSize) + buf := make([]byte, 1024) for { n, err := reader.Read(buf) if err != nil && err != io.EOF { @@ -99,20 +74,30 @@ func getHashFromReader(reader io.Reader) uint64 { func NewFileInfo(reader *tar.Reader, header *tar.Header, path string) FileInfo { if header.Typeflag == tar.TypeDir { return FileInfo{ - Path: path, - TypeFlag: header.Typeflag, - hash: 0, - TarHeader: *header, + Path: path, + TypeFlag: header.Typeflag, + Linkname: header.Linkname, + hash: 0, + Size: header.FileInfo().Size(), + Mode: header.FileInfo().Mode(), + Uid: header.Uid, + Gid: header.Gid, + IsDir: header.FileInfo().IsDir(), } } hash := getHashFromReader(reader) return FileInfo{ - Path: path, - TypeFlag: header.Typeflag, - hash: hash, - TarHeader: *header, + Path: path, + TypeFlag: header.Typeflag, + Linkname: header.Linkname, + hash: hash, + Size: header.FileInfo().Size(), + Mode: header.FileInfo().Mode(), + Uid: header.Uid, + Gid: header.Gid, + IsDir: header.FileInfo().IsDir(), } } @@ -122,10 +107,15 @@ func (data *FileInfo) Copy() *FileInfo { return nil } return &FileInfo{ - Path: data.Path, - TypeFlag: data.TypeFlag, - hash: data.hash, - TarHeader: data.TarHeader, + Path: data.Path, + TypeFlag: data.TypeFlag, + Linkname: data.Linkname, + hash: data.hash, + Size: data.Size, + Mode: data.Mode, + Uid: data.Uid, + Gid: data.Gid, + IsDir: data.IsDir, } } diff --git a/filetree/data_test.go b/filetree/data_test.go index fff79d8..7c9fad7 100644 --- a/filetree/data_test.go +++ b/filetree/data_test.go @@ -6,7 +6,7 @@ import ( func TestAssignDiffType(t *testing.T) { tree := NewFileTree() - node, err := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) + node, _, err := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) if err != nil { t.Errorf("Expected no error from fetching path. got: %v", err) } diff --git a/filetree/efficiency.go b/filetree/efficiency.go index e383b9d..b521bff 100644 --- a/filetree/efficiency.go +++ b/filetree/efficiency.go @@ -6,17 +6,6 @@ import ( "sort" ) -// EfficiencyData represents the storage and reference statistics for a given file tree path. -type EfficiencyData struct { - Path string - Nodes []*FileNode - CumulativeSize int64 - minDiscoveredSize int64 -} - -// EfficiencySlice represents an ordered set of EfficiencyData data structures. -type EfficiencySlice []*EfficiencyData - // Len is required for sorting. func (efs EfficiencySlice) Len() int { return len(efs) @@ -59,19 +48,19 @@ func Efficiency(trees []*FileTree) (float64, EfficiencySlice) { if node.IsWhiteout() { sizer := func(curNode *FileNode) error { - sizeBytes += curNode.Data.FileInfo.TarHeader.FileInfo().Size() + sizeBytes += curNode.Data.FileInfo.Size return nil } - stackedTree := StackRange(trees, 0, currentTree-1) + stackedTree := StackTreeRange(trees, 0, currentTree-1) previousTreeNode, err := stackedTree.GetNode(node.Path()) if err != nil { logrus.Debug(fmt.Sprintf("CurrentTree: %d : %s", currentTree, err)) - } else if previousTreeNode.Data.FileInfo.TarHeader.FileInfo().IsDir() { + } else if previousTreeNode.Data.FileInfo.IsDir { previousTreeNode.VisitDepthChildFirst(sizer, nil) } } else { - sizeBytes = node.Data.FileInfo.TarHeader.FileInfo().Size() + sizeBytes = node.Data.FileInfo.Size } data.CumulativeSize += sizeBytes diff --git a/filetree/efficiency_test.go b/filetree/efficiency_test.go index 4988053..3ec779f 100644 --- a/filetree/efficiency_test.go +++ b/filetree/efficiency_test.go @@ -1,7 +1,6 @@ package filetree import ( - "archive/tar" "testing" ) @@ -11,11 +10,11 @@ func TestEfficencyMap(t *testing.T) { trees[idx] = NewFileTree() } - trees[0].AddPath("/etc/nginx/nginx.conf", FileInfo{TarHeader: tar.Header{Size: 2000}}) - trees[0].AddPath("/etc/nginx/public", FileInfo{TarHeader: tar.Header{Size: 3000}}) + trees[0].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 2000}) + trees[0].AddPath("/etc/nginx/public", FileInfo{Size: 3000}) - trees[1].AddPath("/etc/nginx/nginx.conf", FileInfo{TarHeader: tar.Header{Size: 5000}}) - trees[1].AddPath("/etc/athing", FileInfo{TarHeader: tar.Header{Size: 10000}}) + trees[1].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 5000}) + trees[1].AddPath("/etc/athing", FileInfo{Size: 10000}) trees[2].AddPath("/etc/.wh.nginx", *BlankFileChangeInfo("/etc/.wh.nginx")) diff --git a/filetree/node.go b/filetree/node.go index 233692a..7df04a2 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -22,16 +22,6 @@ var diffTypeColor = map[DiffType]*color.Color{ Unchanged: color.New(color.Reset), } -// FileNode represents a single file, its relation to files beneath it, the tree it exists in, and the metadata of the given file. -type FileNode struct { - Tree *FileTree - Parent *FileNode - Name string - Data NodeData - Children map[string]*FileNode - path string -} - // NewNode creates a new FileNode relative to the given parent node with a payload. func NewNode(parent *FileNode, name string, data FileInfo) (node *FileNode) { node = new(FileNode) @@ -124,8 +114,8 @@ func (node *FileNode) String() string { } display = node.Name - if node.Data.FileInfo.TarHeader.Typeflag == tar.TypeSymlink || node.Data.FileInfo.TarHeader.Typeflag == tar.TypeLink { - display += " → " + node.Data.FileInfo.TarHeader.Linkname + if node.Data.FileInfo.TypeFlag == tar.TypeSymlink || node.Data.FileInfo.TypeFlag == tar.TypeLink { + display += " → " + node.Data.FileInfo.Linkname } return diffTypeColor[node.Data.DiffType].Sprint(display) } @@ -136,25 +126,25 @@ func (node *FileNode) MetadataString() string { return "" } - fileMode := permbits.FileMode(node.Data.FileInfo.TarHeader.FileInfo().Mode()).String() + fileMode := permbits.FileMode(node.Data.FileInfo.Mode).String() dir := "-" - if node.Data.FileInfo.TarHeader.FileInfo().IsDir() { + if node.Data.FileInfo.IsDir { dir = "d" } - user := node.Data.FileInfo.TarHeader.Uid - group := node.Data.FileInfo.TarHeader.Gid + user := node.Data.FileInfo.Uid + group := node.Data.FileInfo.Gid userGroup := fmt.Sprintf("%d:%d", user, group) var sizeBytes int64 if node.IsLeaf() { - sizeBytes = node.Data.FileInfo.TarHeader.FileInfo().Size() + sizeBytes = node.Data.FileInfo.Size } else { sizer := func(curNode *FileNode) error { // don't include file sizes of children that have been removed (unless the node in question is a removed dir, // then show the accumulated size of removed files) if curNode.Data.DiffType != Removed || node.Data.DiffType == Removed { - sizeBytes += curNode.Data.FileInfo.TarHeader.FileInfo().Size() + sizeBytes += curNode.Data.FileInfo.Size } return nil } @@ -264,8 +254,8 @@ func (node *FileNode) deriveDiffType(diffType DiffType) error { if node.IsLeaf() { return node.AssignDiffType(diffType) } - myDiffType := diffType + myDiffType := diffType for _, v := range node.Children { myDiffType = myDiffType.merge(v.Data.DiffType) @@ -274,19 +264,14 @@ func (node *FileNode) deriveDiffType(diffType DiffType) error { return node.AssignDiffType(myDiffType) } -// AssignDiffType will assign the given DiffType to this node, possible affecting child nodes. +// AssignDiffType will assign the given DiffType to this node, possibly affecting child nodes. func (node *FileNode) AssignDiffType(diffType DiffType) error { var err error - // todo, this is an indicator that the root node approach isn't working - if node.Path() == "/" { - return nil - } - node.Data.DiffType = diffType - // if we've removed this node, then all children have been removed as well if diffType == Removed { + // if we've removed this node, then all children have been removed as well for _, child := range node.Children { err = child.AssignDiffType(diffType) if err != nil { @@ -294,6 +279,7 @@ func (node *FileNode) AssignDiffType(diffType DiffType) error { } } } + return nil } diff --git a/filetree/node_test.go b/filetree/node_test.go index db39d7d..9d9bcd9 100644 --- a/filetree/node_test.go +++ b/filetree/node_test.go @@ -1,7 +1,6 @@ package filetree import ( - "archive/tar" "testing" ) @@ -84,7 +83,7 @@ func TestRemoveChild(t *testing.T) { func TestPath(t *testing.T) { expected := "/etc/nginx/nginx.conf" tree := NewFileTree() - node, _ := tree.AddPath(expected, FileInfo{}) + node, _, _ := tree.AddPath(expected, FileInfo{}) actual := node.Path() if expected != actual { @@ -94,9 +93,9 @@ func TestPath(t *testing.T) { func TestIsWhiteout(t *testing.T) { tree1 := NewFileTree() - p1, _ := tree1.AddPath("/etc/nginx/public1", FileInfo{}) - p2, _ := tree1.AddPath("/etc/nginx/.wh.public2", FileInfo{}) - p3, _ := tree1.AddPath("/etc/nginx/public3/.wh..wh..opq", FileInfo{}) + p1, _, _ := tree1.AddPath("/etc/nginx/public1", FileInfo{}) + p2, _, _ := tree1.AddPath("/etc/nginx/.wh.public2", FileInfo{}) + p3, _, _ := tree1.AddPath("/etc/nginx/public3/.wh..wh..opq", FileInfo{}) if p1.IsWhiteout() != false { t.Errorf("Expected path '%s' to **not** be a whiteout file", p1.Name) @@ -113,15 +112,13 @@ func TestIsWhiteout(t *testing.T) { func TestDiffTypeFromAddedChildren(t *testing.T) { tree := NewFileTree() - node, _ := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) + node, _, _ := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) node.Data.DiffType = Unchanged - info1 := BlankFileChangeInfo("/usr/bin") - node, _ = tree.AddPath("/usr/bin", *info1) + node, _, _ = tree.AddPath("/usr/bin", *BlankFileChangeInfo("/usr/bin")) node.Data.DiffType = Added - info2 := BlankFileChangeInfo("/usr/bin2") - node, _ = tree.AddPath("/usr/bin2", *info2) + node, _, _ = tree.AddPath("/usr/bin2", *BlankFileChangeInfo("/usr/bin2")) node.Data.DiffType = Removed tree.Root.Children["usr"].deriveDiffType(Unchanged) @@ -132,14 +129,14 @@ func TestDiffTypeFromAddedChildren(t *testing.T) { } func TestDiffTypeFromRemovedChildren(t *testing.T) { tree := NewFileTree() - node, _ := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) + node, _, _ := tree.AddPath("/usr", *BlankFileChangeInfo("/usr")) info1 := BlankFileChangeInfo("/usr/.wh.bin") - node, _ = tree.AddPath("/usr/.wh.bin", *info1) + node, _, _ = tree.AddPath("/usr/.wh.bin", *info1) node.Data.DiffType = Removed info2 := BlankFileChangeInfo("/usr/.wh.bin2") - node, _ = tree.AddPath("/usr/.wh.bin2", *info2) + node, _, _ = tree.AddPath("/usr/.wh.bin2", *info2) node.Data.DiffType = Removed tree.Root.Children["usr"].deriveDiffType(Unchanged) @@ -152,9 +149,9 @@ func TestDiffTypeFromRemovedChildren(t *testing.T) { func TestDirSize(t *testing.T) { tree1 := NewFileTree() - tree1.AddPath("/etc/nginx/public1", FileInfo{TarHeader: tar.Header{Size: 100}}) - tree1.AddPath("/etc/nginx/thing1", FileInfo{TarHeader: tar.Header{Size: 200}}) - tree1.AddPath("/etc/nginx/public3/thing2", FileInfo{TarHeader: tar.Header{Size: 300}}) + tree1.AddPath("/etc/nginx/public1", FileInfo{Size: 100}) + tree1.AddPath("/etc/nginx/thing1", FileInfo{Size: 200}) + tree1.AddPath("/etc/nginx/public3/thing2", FileInfo{Size: 300}) node, _ := tree1.GetNode("/etc/nginx") expected, actual := "---------- 0:0 600 B ", node.MetadataString() diff --git a/filetree/tree.go b/filetree/tree.go index d87e7ec..4255ef3 100644 --- a/filetree/tree.go +++ b/filetree/tree.go @@ -20,15 +20,6 @@ const ( collapsedItem = "⊕ " ) -// FileTree represents a set of files, directories, and their relations. -type FileTree struct { - Root *FileNode - Size int - FileSize uint64 - Name string - Id uuid.UUID -} - // NewFileTree creates an empty FileTree func NewFileTree() (tree *FileTree) { tree = new(FileTree) @@ -179,7 +170,7 @@ func (tree *FileTree) Stack(upper *FileTree) error { return fmt.Errorf("cannot remove node %s: %v", node.Path(), err.Error()) } } else { - newNode, err := tree.AddPath(node.Path(), node.Data.FileInfo) + newNode, _, err := tree.AddPath(node.Path(), node.Data.FileInfo) if err != nil { return fmt.Errorf("cannot add node %s: %v", newNode.Path(), err.Error()) } @@ -206,9 +197,10 @@ func (tree *FileTree) GetNode(path string) (*FileNode, error) { } // AddPath adds a new node to the tree with the given payload -func (tree *FileTree) AddPath(path string, data FileInfo) (*FileNode, error) { +func (tree *FileTree) AddPath(path string, data FileInfo) (*FileNode, []*FileNode, error) { nodeNames := strings.Split(strings.Trim(path, "/"), "/") node := tree.Root + addedNodes := make([]*FileNode, 0) for idx, name := range nodeNames { if name == "" { continue @@ -220,10 +212,11 @@ func (tree *FileTree) AddPath(path string, data FileInfo) (*FileNode, error) { // don't attach the payload. The payload is destined for the // Path's end node, not any intermediary node. node = node.AddChild(name, FileInfo{}) + addedNodes = append(addedNodes, node) if node == nil { // the child could not be added - return node, fmt.Errorf(fmt.Sprintf("could not add child node '%s'", name)) + return node, addedNodes, fmt.Errorf(fmt.Sprintf("could not add child node '%s'", name)) } } @@ -233,7 +226,7 @@ func (tree *FileTree) AddPath(path string, data FileInfo) (*FileNode, error) { } } - return node, nil + return node, addedNodes, nil } // RemovePath removes a node from the tree given its path. @@ -245,10 +238,18 @@ func (tree *FileTree) RemovePath(path string) error { return node.Remove() } +type compareMark struct { + node *FileNode + tentative DiffType + final DiffType +} + // Compare marks the FileNodes in the owning (lower) tree with DiffType annotations when compared to the given (upper) tree. func (tree *FileTree) Compare(upper *FileTree) error { // always compare relative to the original, unaltered tree. - originalTree := tree.Copy() + originalTree := tree + + modifications := make([]compareMark, 0) graft := func(upperNode *FileNode) error { if upperNode.IsWhiteout() { @@ -257,27 +258,46 @@ func (tree *FileTree) Compare(upper *FileTree) error { return fmt.Errorf("cannot remove upperNode %s: %v", upperNode.Path(), err.Error()) } } else { - // compare against the original tree (to ensure new parents with new children are captured as new instead of modified) + // note: since we are not comparing against the original tree (copying the tree is expensive) we may mark the parent + // of an added node incorrectly as modified. This will be corrected later. originalLowerNode, _ := originalTree.GetNode(upperNode.Path()) if originalLowerNode == nil { - newNode, err := tree.AddPath(upperNode.Path(), upperNode.Data.FileInfo) + _, newNodes, err := tree.AddPath(upperNode.Path(), upperNode.Data.FileInfo) if err != nil { return fmt.Errorf("cannot add new upperNode %s: %v", upperNode.Path(), err.Error()) } - newNode.AssignDiffType(Added) + for idx := len(newNodes) - 1; idx >= 0; idx-- { + newNode := newNodes[idx] + modifications = append(modifications, compareMark{node: newNode, tentative: -1, final: Added}) + } + } else { // check the tree for comparison markings lowerNode, _ := tree.GetNode(upperNode.Path()) - diffType := lowerNode.compare(upperNode) - return lowerNode.deriveDiffType(diffType) + modifications = append(modifications, compareMark{node: lowerNode, tentative: diffType, final: -1}) } } return nil } // we must visit from the leaves upwards to ensure that diff types can be derived from and assigned to children - return upper.VisitDepthChildFirst(graft, nil) + err := upper.VisitDepthChildFirst(graft, nil) + if err != nil { + return err + } + + // take note of the comparison results on each note in the owning tree + for _, pair := range modifications { + if pair.final > 0 { + pair.node.AssignDiffType(pair.final) + } else { + if pair.node.Data.DiffType == Unchanged { + pair.node.deriveDiffType(pair.tentative) + } + } + } + return nil } // markRemoved annotates the FileNode at the given path as Removed. @@ -289,8 +309,9 @@ func (tree *FileTree) markRemoved(path string) error { return node.AssignDiffType(Removed) } -// StackRange combines an array of trees into a single tree -func StackRange(trees []*FileTree, start, stop int) *FileTree { +// StackTreeRange combines an array of trees into a single tree +func StackTreeRange(trees []*FileTree, start, stop int) *FileTree { + tree := trees[0].Copy() for idx := start; idx <= stop; idx++ { err := tree.Stack(trees[idx]) @@ -298,6 +319,5 @@ func StackRange(trees []*FileTree, start, stop int) *FileTree { logrus.Debug("could not stack tree range:", err) } } - return tree } diff --git a/filetree/tree_test.go b/filetree/tree_test.go index 5002dc6..92c83f2 100644 --- a/filetree/tree_test.go +++ b/filetree/tree_test.go @@ -488,7 +488,7 @@ func TestStackRange(t *testing.T) { upperTree.AddPath(value, fakeData) } trees := []*FileTree{lowerTree, upperTree, tree} - StackRange(trees, 0, 2) + StackTreeRange(trees, 0, 2) } func TestRemoveOnIterate(t *testing.T) { @@ -502,7 +502,7 @@ func TestRemoveOnIterate(t *testing.T) { TypeFlag: 1, hash: 123, } - node, err := tree.AddPath(value, fakeData) + node, _, err := tree.AddPath(value, fakeData) if err == nil && stringInSlice(node.Path(), []string{"/etc"}) { node.Data.ViewInfo.Hidden = true } diff --git a/filetree/types.go b/filetree/types.go new file mode 100644 index 0000000..d288c4e --- /dev/null +++ b/filetree/types.go @@ -0,0 +1,65 @@ +package filetree + +import ( + "github.com/google/uuid" + "os" +) + +// FileTree represents a set of files, directories, and their relations. +type FileTree struct { + Root *FileNode + Size int + FileSize uint64 + Name string + Id uuid.UUID +} + +// FileNode represents a single file, its relation to files beneath it, the tree it exists in, and the metadata of the given file. +type FileNode struct { + Tree *FileTree + Parent *FileNode + Name string + Data NodeData + Children map[string]*FileNode + path string +} + +// NodeData is the payload for a FileNode +type NodeData struct { + ViewInfo ViewInfo + FileInfo FileInfo + DiffType DiffType +} + +// ViewInfo contains UI specific detail for a specific FileNode +type ViewInfo struct { + Collapsed bool + Hidden bool +} + +// FileInfo contains tar metadata for a specific FileNode +type FileInfo struct { + Path string + TypeFlag byte + Linkname string + hash uint64 + Size int64 + Mode os.FileMode + Uid int + Gid int + IsDir bool +} + +// DiffType defines the comparison result between two FileNodes +type DiffType int + +// EfficiencyData represents the storage and reference statistics for a given file tree path. +type EfficiencyData struct { + Path string + Nodes []*FileNode + CumulativeSize int64 + minDiscoveredSize int64 +} + +// EfficiencySlice represents an ordered set of EfficiencyData data structures. +type EfficiencySlice []*EfficiencyData diff --git a/image/docker_image.go b/image/docker_image.go index 0e3c309..adddefb 100644 --- a/image/docker_image.go +++ b/image/docker_image.go @@ -214,11 +214,10 @@ func (image *dockerImageAnalyzer) processLayerTar(name string, layerIdx uint, re shortName := name[:15] pb := utils.NewProgressBar(int64(len(fileInfos)), 30) for idx, element := range fileInfos { - tree.FileSize += uint64(element.TarHeader.FileInfo().Size()) - _, err := tree.AddPath(element.Path, element) - if err != nil { - return err - } + tree.FileSize += uint64(element.Size) + + // todo: we should check for errors but also allow whiteout files to be not be added (thus not error out) + tree.AddPath(element.Path, element) if pb.Update(int64(idx)) { message = fmt.Sprintf(" ├─ %s %s : %s", title, shortName, pb.String()) @@ -238,12 +237,9 @@ func (image *dockerImageAnalyzer) getFileList(tarReader *tar.Reader) ([]filetree for { header, err := tarReader.Next() - if err == io.EOF { break - } - - if err != nil { + } else if err != nil { fmt.Println(err) utils.Exit(1) } diff --git a/ui/filetreeview.go b/ui/filetreeview.go index 1a5459f..db03bad 100644 --- a/ui/filetreeview.go +++ b/ui/filetreeview.go @@ -30,6 +30,7 @@ type FileTreeView struct { ModelTree *filetree.FileTree ViewTree *filetree.FileTree RefTrees []*filetree.FileTree + cache filetree.TreeCache HiddenDiffTypes []bool TreeIndex uint bufferIndex uint @@ -46,7 +47,7 @@ type FileTreeView struct { } // NewFileTreeView creates a new view object attached the the global [gocui] screen object. -func NewFileTreeView(name string, gui *gocui.Gui, tree *filetree.FileTree, refTrees []*filetree.FileTree) (treeView *FileTreeView) { +func NewFileTreeView(name string, gui *gocui.Gui, tree *filetree.FileTree, refTrees []*filetree.FileTree, cache filetree.TreeCache) (treeView *FileTreeView) { treeView = new(FileTreeView) // populate main fields @@ -54,6 +55,7 @@ func NewFileTreeView(name string, gui *gocui.Gui, tree *filetree.FileTree, refTr treeView.gui = gui treeView.ModelTree = tree treeView.RefTrees = refTrees + treeView.cache = cache treeView.HiddenDiffTypes = make([]bool, 4) hiddenTypes := viper.GetStringSlice("diff.hide") @@ -184,11 +186,7 @@ func (view *FileTreeView) setTreeByLayer(bottomTreeStart, bottomTreeStop, topTre if topTreeStop > len(view.RefTrees)-1 { return fmt.Errorf("invalid layer index given: %d of %d", topTreeStop, len(view.RefTrees)-1) } - newTree := filetree.StackRange(view.RefTrees, bottomTreeStart, bottomTreeStop) - - for idx := topTreeStart; idx <= topTreeStop; idx++ { - newTree.Compare(view.RefTrees[idx]) - } + newTree := view.cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop) // preserve view state on copy visitor := func(node *filetree.FileNode) error { @@ -320,7 +318,7 @@ func (view *FileTreeView) CursorRight() error { if node == nil { return nil } - if !node.Data.FileInfo.TarHeader.FileInfo().IsDir() { + if !node.Data.FileInfo.IsDir { return nil } if len(node.Children) == 0 { @@ -459,7 +457,7 @@ func filterRegex() *regexp.Regexp { return nil } filterString := strings.TrimSpace(Views.Filter.view.Buffer()) - if len(filterString) < 1 { + if len(filterString) == 0 { return nil } diff --git a/ui/ui.go b/ui/ui.go index 6ad30e8..dae772c 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -15,7 +15,8 @@ import ( const debug = false -// var profileObj = profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.NoShutdownHook) +// var profileObj = profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook) +// var onExit func() // debugPrint writes the given string to the debug pane (if the debug pane is enabled) func debugPrint(s string) { @@ -143,6 +144,7 @@ func CursorUp(g *gocui.Gui, v *gocui.View) error { func quit(g *gocui.Gui, v *gocui.View) error { // profileObj.Stop() + // onExit() return gocui.ErrQuit } @@ -301,7 +303,7 @@ func renderStatusOption(control, title string, selected bool) string { } // Run is the UI entrypoint. -func Run(analysis *image.AnalysisResult) { +func Run(analysis *image.AnalysisResult, cache filetree.TreeCache) { Formatting.Selected = color.New(color.ReverseVideo, color.Bold).SprintFunc() Formatting.Header = color.New(color.Bold).SprintFunc() @@ -328,7 +330,7 @@ func Run(analysis *image.AnalysisResult) { Views.Layer = NewLayerView("side", g, analysis.Layers) Views.lookup[Views.Layer.Name] = Views.Layer - Views.Tree = NewFileTreeView("main", g, filetree.StackRange(analysis.RefTrees, 0, 0), analysis.RefTrees) + Views.Tree = NewFileTreeView("main", g, filetree.StackTreeRange(analysis.RefTrees, 0, 0), analysis.RefTrees, cache) Views.lookup[Views.Tree.Name] = Views.Tree Views.Status = NewStatusView("status", g) @@ -344,6 +346,12 @@ func Run(analysis *image.AnalysisResult) { //g.Mouse = true g.SetManagerFunc(layout) + // var profileObj = profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.NoShutdownHook) + // + // onExit = func() { + // profileObj.Stop() + // } + // perform the first update and render now that all resources have been loaded Update() Render()