From c599ca5ad20bf52b6fde06f9462d8830cab90b7f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Sat, 6 Oct 2018 09:45:08 -0400 Subject: [PATCH] a few low hanging perf improvements (#16) --- Makefile | 2 +- cmd/analyze.go | 20 ------- cmd/build.go | 20 ------- cmd/root.go | 20 ------- filetree/node.go | 136 ++++++++++++++++++++++++++++++++--------------- filetree/tree.go | 84 +++++++++++------------------ image/image.go | 70 ++++++++---------------- image/layer.go | 44 +++++++++++++++ main.go | 5 +- ui/layerview.go | 20 ++++++- ui/ui.go | 27 +++++++++- 11 files changed, 240 insertions(+), 208 deletions(-) create mode 100644 image/layer.go diff --git a/Makefile b/Makefile index 0aea4cd..e79f5a2 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ install: go install ./... test: build - go test -v ./... + go test -cover -v ./... lint: build golint -set_exit_status $$(go list ./...) diff --git a/cmd/analyze.go b/cmd/analyze.go index 88745e3..9018784 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -1,23 +1,3 @@ -// Copyright © 2018 Alex Goodman -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package cmd import ( diff --git a/cmd/build.go b/cmd/build.go index 99ebdd8..7ccdbfc 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -1,23 +1,3 @@ -// Copyright © 2018 Alex Goodman -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package cmd import ( diff --git a/cmd/root.go b/cmd/root.go index 91d46a7..f8804b1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,23 +1,3 @@ -// Copyright © 2018 Alex Goodman -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - package cmd import ( diff --git a/filetree/node.go b/filetree/node.go index 71507ed..4231cae 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -15,12 +15,20 @@ const ( AttributeFormat = "%s%s %10s %10s " ) +var diffTypeColor = map[DiffType]*color.Color { + Added: color.New(color.FgGreen), + Removed: color.New(color.FgRed), + Changed: color.New(color.FgYellow), + Unchanged: color.New(color.Reset), +} + type FileNode struct { Tree *FileTree Parent *FileNode Name string Data NodeData Children map[string]*FileNode + path string } func NewNode(parent *FileNode, name string, data FileInfo) (node *FileNode) { @@ -37,6 +45,59 @@ func NewNode(parent *FileNode, name string, data FileInfo) (node *FileNode) { return node } +// todo: make more performant +// todo: rewrite with visitor functions +func (node *FileNode) renderTreeLine(spaces []bool, last bool, collapsed bool) string { + var otherBranches string + for _, space := range spaces { + if space { + otherBranches += noBranchSpace + } else { + otherBranches += branchSpace + } + } + + thisBranch := middleItem + if last { + thisBranch = lastItem + } + + collapsedIndicator := uncollapsedItem + if collapsed { + collapsedIndicator = collapsedItem + } + + return otherBranches + thisBranch + collapsedIndicator + node.String() + newLine +} + +// todo: make more performant +// todo: rewrite with visitor functions +func (node *FileNode) renderStringTree(spaces []bool, showAttributes bool, depth int) string { + var result string + var keys []string + for key := range node.Children { + keys = append(keys, key) + } + sort.Strings(keys) + for idx, name := range keys { + child := node.Children[name] + if child.Data.ViewInfo.Hidden { + continue + } + last := idx == (len(node.Children) - 1) + showCollapsed := child.Data.ViewInfo.Collapsed && len(child.Children) > 0 + if showAttributes { + result += child.MetadataString() + " " + } + result += child.renderTreeLine(spaces, last, showCollapsed) + if len(child.Children) > 0 && !child.Data.ViewInfo.Collapsed { + spacesChild := append(spaces, last) + result += child.renderStringTree(spacesChild, showAttributes, depth+1) + } + } + return result +} + func (node *FileNode) Copy(parent *FileNode) *FileNode { newNode := NewNode(parent, node.Name, node.Data.FileInfo) newNode.Data.ViewInfo = node.Data.ViewInfo @@ -73,47 +134,22 @@ func (node *FileNode) Remove() error { } func (node *FileNode) String() string { - var style *color.Color var display string if node == nil { return "" } - switch node.Data.DiffType { - case Added: - style = color.New(color.FgGreen) - case Removed: - style = color.New(color.FgRed) - case Changed: - style = color.New(color.FgYellow) - case Unchanged: - style = color.New(color.Reset) - default: - style = color.New(color.BgMagenta) - } + display = node.Name if node.Data.FileInfo.TarHeader.Typeflag == tar.TypeSymlink || node.Data.FileInfo.TarHeader.Typeflag == tar.TypeLink { display += " → " + node.Data.FileInfo.TarHeader.Linkname } - return style.Sprint(display) + return diffTypeColor[node.Data.DiffType].Sprint(display) } func (node *FileNode) MetadataString() string { - var style *color.Color if node == nil { return "" } - switch node.Data.DiffType { - case Added: - style = color.New(color.FgGreen) - case Removed: - style = color.New(color.FgRed) - case Changed: - style = color.New(color.FgYellow) - case Unchanged: - style = color.New(color.Reset) - default: - style = color.New(color.BgMagenta) - } fileMode := permbits.FileMode(node.Data.FileInfo.TarHeader.FileInfo().Mode()).String() dir := "-" @@ -143,7 +179,7 @@ func (node *FileNode) MetadataString() string { size := humanize.Bytes(uint64(sizeBytes)) - return style.Sprint(fmt.Sprintf(AttributeFormat, dir, fileMode, userGroup, size)) + return diffTypeColor[node.Data.DiffType].Sprint(fmt.Sprintf(AttributeFormat, dir, fileMode, userGroup, size)) } func (node *FileNode) VisitDepthChildFirst(visiter Visiter, evaluator VisitEvaluator) error { @@ -205,24 +241,40 @@ func (node *FileNode) IsWhiteout() bool { return strings.HasPrefix(node.Name, whiteoutPrefix) } +// todo: make path() more efficient, similar to so (buggy): +// func (node *FileNode) Path() string { +// if node.path == "" { +// path := "/" +// +// if node.Parent != nil { +// path = node.Parent.Path() +// } +// node.path = path + "/" + strings.TrimPrefix(node.Name, whiteoutPrefix) +// } +// return node.path +// } + func (node *FileNode) Path() string { - path := []string{} - curNode := node - for { - if curNode.Parent == nil { - break - } + if node.path == "" { + path := []string{} + curNode := node + for { + if curNode.Parent == nil { + break + } - name := curNode.Name - if curNode == node { - // white out prefixes are fictitious on leaf nodes - name = strings.TrimPrefix(name, whiteoutPrefix) - } + name := curNode.Name + if curNode == node { + // white out prefixes are fictitious on leaf nodes + name = strings.TrimPrefix(name, whiteoutPrefix) + } - path = append([]string{name}, path...) - curNode = curNode.Parent + path = append([]string{name}, path...) + curNode = curNode.Parent + } + node.path = "/" + strings.Join(path, "/") } - return "/" + strings.Join(path, "/") + return node.path } func (node *FileNode) IsLeaf() bool { diff --git a/filetree/tree.go b/filetree/tree.go index 155df5b..53081cf 100644 --- a/filetree/tree.go +++ b/filetree/tree.go @@ -2,8 +2,8 @@ package filetree import ( "fmt" - "sort" "strings" + "github.com/satori/go.uuid" ) const ( @@ -21,6 +21,7 @@ type FileTree struct { Root *FileNode Size int Name string + Id uuid.UUID } func NewFileTree() (tree *FileTree) { @@ -29,63 +30,12 @@ func NewFileTree() (tree *FileTree) { tree.Root = new(FileNode) tree.Root.Tree = tree tree.Root.Children = make(map[string]*FileNode) + tree.Id = uuid.Must(uuid.NewV4()) return tree } func (tree *FileTree) String(showAttributes bool) string { - var renderTreeLine func(string, []bool, bool, bool) string - var walkTree func(*FileNode, []bool, int) string - - renderTreeLine = func(nodeText string, spaces []bool, last bool, collapsed bool) string { - var otherBranches string - for _, space := range spaces { - if space { - otherBranches += noBranchSpace - } else { - otherBranches += branchSpace - } - } - - thisBranch := middleItem - if last { - thisBranch = lastItem - } - - collapsedIndicator := uncollapsedItem - if collapsed { - collapsedIndicator = collapsedItem - } - - return otherBranches + thisBranch + collapsedIndicator + nodeText + newLine - } - - walkTree = func(node *FileNode, spaces []bool, depth int) string { - var result string - var keys []string - for key := range node.Children { - keys = append(keys, key) - } - sort.Strings(keys) - for idx, name := range keys { - child := node.Children[name] - if child.Data.ViewInfo.Hidden { - continue - } - last := idx == (len(node.Children) - 1) - showCollapsed := child.Data.ViewInfo.Collapsed && len(child.Children) > 0 - if showAttributes { - result += child.MetadataString() + " " - } - result += renderTreeLine(child.String(), spaces, last, showCollapsed) - if len(child.Children) > 0 && !child.Data.ViewInfo.Collapsed { - spacesChild := append(spaces, last) - result += walkTree(child, spacesChild, depth+1) - } - } - return result - } - - return walkTree(tree.Root, []bool{}, 0) + return tree.Root.renderStringTree([]bool{}, showAttributes, 0) } func (tree *FileTree) Copy() *FileTree { @@ -214,11 +164,37 @@ func (tree *FileTree) MarkRemoved(path string) error { return node.AssignDiffType(Removed) } +// memoize StackRange for performance +type stackRangeCacheKey struct { + // Ids mapset.Set + start, stop int +} + +var stackRangeCache = make(map[stackRangeCacheKey]*FileTree) + func StackRange(trees []*FileTree, start, stop int) *FileTree { + + // var ids []interface{} + // + // for _, tree := range trees { + // ids = append(ids, tree.Id) + // } +//mapset.NewSetFromSlice(ids) +// key := stackRangeCacheKey{start, stop} +// +// +// cachedResult, ok := stackRangeCache[key] +// if ok { +// return cachedResult +// } + tree := trees[0].Copy() for idx := start; idx <= stop; idx++ { tree.Stack(trees[idx]) } + + // stackRangeCache[key] = tree + return tree } diff --git a/image/image.go b/image/image.go index b45c980..c3bb31d 100644 --- a/image/image.go +++ b/image/image.go @@ -12,12 +12,9 @@ import ( "path/filepath" "strings" - "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" - humanize "github.com/dustin/go-humanize" "github.com/wagoodman/dive/filetree" "golang.org/x/net/context" - "strconv" ) const ( @@ -43,42 +40,12 @@ func NewManifest(reader *tar.Reader, header *tar.Header) ImageManifest { if err != nil && err != io.EOF { panic(err) } - var m []ImageManifest - err = json.Unmarshal(manifestBytes, &m) + var manifest []ImageManifest + err = json.Unmarshal(manifestBytes, &manifest) if err != nil { panic(err) } - return m[0] -} - -type Layer struct { - TarPath string - History image.HistoryResponseItem - Index int - Tree *filetree.FileTree - RefTrees []*filetree.FileTree -} - -func (layer *Layer) Id() string { - rangeBound := 25 - if length := len(layer.History.ID); length < 25 { - rangeBound = length - } - id := layer.History.ID[0:rangeBound] - if len(layer.History.Tags) > 0 { - id = "[" + strings.Join(layer.History.Tags, ",") + "]" - } - return id -} - -func (layer *Layer) String() string { - - return fmt.Sprintf(LayerFormat, - layer.Id(), - strconv.Itoa(int(100.0*filetree.EfficiencyScore(layer.RefTrees[:layer.Index+1]))) + "%", - //"100%", - humanize.Bytes(uint64(layer.History.Size)), - strings.TrimPrefix(layer.History.CreatedBy, "/bin/sh -c ")) + return manifest[0] } func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) { @@ -87,10 +54,15 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) { var trees []*filetree.FileTree = make([]*filetree.FileTree, 0) // save this image to disk temporarily to get the content info - imageTarPath, tmpDir := saveImage(imageID) - defer os.RemoveAll(tmpDir) + fmt.Println("Fetching image...") + // imageTarPath, tmpDir := saveImage(imageID) + imageTarPath := "/tmp/dive031537738/image.tar" + // tmpDir := "/tmp/dive031537738" + // fmt.Println(tmpDir) + // defer os.RemoveAll(tmpDir) // read through the image contents and build a tree + fmt.Println("Reading image...") tarFile, err := os.Open(imageTarPath) if err != nil { fmt.Println(err) @@ -135,13 +107,14 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) { } // build the content tree + fmt.Println("Building tree...") for _, treeName := range manifest.LayerTarPaths { trees = append(trees, layerMap[treeName]) } // get the history of this image ctx := context.Background() - dockerClient, err := client.NewEnvClient() + dockerClient, err := client.NewClientWithOpts() if err != nil { panic(err) } @@ -149,13 +122,14 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) { history, err := dockerClient.ImageHistory(ctx, imageID) // build the layers array - layers := make([]*Layer, len(history)-1) - for idx := 0; idx < len(layers); idx++ { - layers[idx] = new(Layer) - layers[idx].History = history[idx] - layers[idx].Index = idx - layers[idx].Tree = trees[idx] - layers[idx].RefTrees = trees + layers := make([]*Layer, len(trees)) + for idx := 0; idx < len(trees); idx++ { + layers[idx] = &Layer{ + History: history[idx], + Index: idx, + Tree: trees[idx], + RefTrees: trees, + } if len(manifest.LayerTarPaths) > idx { layers[idx].TarPath = manifest.LayerTarPaths[idx] } @@ -166,7 +140,7 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) { func saveImage(imageID string) (string, string) { ctx := context.Background() - dockerClient, err := client.NewEnvClient() + dockerClient, err := client.NewClientWithOpts() if err != nil { panic(err) } @@ -175,7 +149,7 @@ func saveImage(imageID string) (string, string) { check(err) defer readCloser.Close() - tmpDir, err := ioutil.TempDir("", "docker-image-explorer") + tmpDir, err := ioutil.TempDir("", "dive") check(err) imageTarPath := filepath.Join(tmpDir, "image.tar") diff --git a/image/layer.go b/image/layer.go new file mode 100644 index 0000000..d342fcf --- /dev/null +++ b/image/layer.go @@ -0,0 +1,44 @@ +package image + +import ( + "github.com/docker/docker/api/types/image" + "github.com/wagoodman/dive/filetree" + "strings" + "fmt" + "strconv" + "github.com/dustin/go-humanize" +) + +type Layer struct { + TarPath string + History image.HistoryResponseItem + Index int + Tree *filetree.FileTree + RefTrees []*filetree.FileTree +} + +func (layer *Layer) Id() string { + rangeBound := 25 + if length := len(layer.History.ID); length < 25 { + rangeBound = length + } + id := layer.History.ID[0:rangeBound] + + // show the tagged image as the last layer + // if len(layer.History.Tags) > 0 { + // id = "[" + strings.Join(layer.History.Tags, ",") + "]" + // } + + return id +} + +func (layer *Layer) String() string { + + return fmt.Sprintf(LayerFormat, + layer.Id(), + strconv.Itoa(int(100.0*filetree.EfficiencyScore(layer.RefTrees[:layer.Index+1]))) + "%", + //"100%", + humanize.Bytes(uint64(layer.History.Size)), + strings.TrimPrefix(layer.History.CreatedBy, "/bin/sh -c ")) +} + diff --git a/main.go b/main.go index d6e186f..4f4451c 100644 --- a/main.go +++ b/main.go @@ -20,8 +20,11 @@ package main -import "github.com/wagoodman/dive/cmd" +import ( + "github.com/wagoodman/dive/cmd" +) func main() { cmd.Execute() } + diff --git a/ui/layerview.go b/ui/layerview.go index d83822c..6bb95b5 100644 --- a/ui/layerview.go +++ b/ui/layerview.go @@ -144,8 +144,15 @@ func (view *LayerView) Render() error { layerStr := layer.String() if idx == 0 { + var layerId string + if len(layer.History.ID) >= 25 { + layerId = layer.History.ID[0:25] + } else { + layerId = fmt.Sprintf("%-25s", layer.History.ID) + } + // TODO: add size - layerStr = fmt.Sprintf(image.LayerFormat, layer.History.ID[0:25], "", "", "FROM "+layer.Id()) + layerStr = fmt.Sprintf(image.LayerFormat, layerId, "", "", "FROM "+layer.Id()) } compareBar := view.renderCompareBar(idx) @@ -170,6 +177,7 @@ func (view *LayerView) CursorDown() error { view.LayerIndex++ Views.Tree.setTreeByLayer(view.getCompareIndexes()) view.Render() + // debugPrint(fmt.Sprintf("%d",len(filetree.Cache))) } } return nil @@ -182,11 +190,21 @@ func (view *LayerView) CursorUp() error { view.LayerIndex-- Views.Tree.setTreeByLayer(view.getCompareIndexes()) view.Render() + // debugPrint(fmt.Sprintf("%d",len(filetree.Cache))) } } return nil } +func (view *LayerView) SetCursor(layer int) error { + // view.view.SetCursor(0, layer) + view.LayerIndex = layer + Views.Tree.setTreeByLayer(view.getCompareIndexes()) + view.Render() + + return nil +} + func (view *LayerView) KeyHelp() string { return renderStatusOption("^L","Layer changes", view.CompareMode == CompareLayer) + renderStatusOption("^A","All changes", view.CompareMode == CompareAll) diff --git a/ui/ui.go b/ui/ui.go index f9458ec..90c7d04 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -9,9 +9,13 @@ import ( "github.com/wagoodman/dive/filetree" "github.com/wagoodman/dive/image" "github.com/fatih/color" + "os" + "runtime" + "runtime/pprof" ) -const debug = false +const debug = true +const profile = false func debugPrint(s string) { if debug && Views.Tree != nil && Views.Tree.gui != nil { @@ -120,7 +124,18 @@ func CursorUp(g *gocui.Gui, v *gocui.View) error { return nil } + +var cpuProfilePath *os.File +var memoryProfilePath *os.File + func quit(g *gocui.Gui, v *gocui.View) error { + if profile { + pprof.StopCPUProfile() + runtime.GC() // get up-to-date statistics + pprof.WriteHeapProfile(memoryProfilePath) + memoryProfilePath.Close() + cpuProfilePath.Close() + } return gocui.ErrQuit } @@ -246,6 +261,7 @@ func renderStatusOption(control, title string, selected bool) string { } func Run(layers []*image.Layer, refTrees []*filetree.FileTree) { + Formatting.Selected = color.New(color.ReverseVideo, color.Bold).SprintFunc() Formatting.Header = color.New(color.Bold).SprintFunc() Formatting.StatusSelected = color.New(color.BgMagenta, color.FgWhite).SprintFunc() @@ -279,10 +295,19 @@ func Run(layers []*image.Layer, refTrees []*filetree.FileTree) { //g.Mouse = true g.SetManagerFunc(layout) + // let the default position of the cursor be the last layer + // Views.Layer.SetCursor(len(Views.Layer.Layers)-1) + if err := keybindings(g); err != nil { log.Panicln(err) } + if profile { + os.Create("cpu.pprof") + os.Create("mem.pprof") + pprof.StartCPUProfile(cpuProfilePath) + } + if err := g.MainLoop(); err != nil && err != gocui.ErrQuit { log.Panicln(err) }