norohind/dive-zfs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge first: Diff trees (#2)

Browse Source
This commit is contained in:
William Murphy 2018-05-31 16:59:45 -04:00 committed by Alex Goodman
parent 4942a9c1ed
commit 18bb252d10
5 changed files with 437 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tar-read.go
View File

@ -28,7 +28,9 @@ func main() {
tarReader := tar.NewReader(f)
targetName := "manifest.json"
var m Manifest
var trees []*FileTree
var layerMap map[string]*FileTree
layerMap = make(map[string]*FileTree)
// var trees []*FileTree
for {
header, err := tarReader.Next()
@ -54,12 +56,13 @@ func main() {
if strings.HasSuffix(name, "layer.tar") {
fmt.Println("Containing:")
tree := NewTree()
tree.name = strings.TrimSuffix(name, "layer.tar")
tree.name = name
fmt.Printf("%s\n", tree.name)
fileInfos := getFileList(tarReader, header)
for _, element := range fileInfos {
tree.AddPath(element.path, element)
tree.AddPath(element.path, &element)
}
trees = append(trees, tree)
layerMap[tree.name] = tree
}
default:
fmt.Printf("%s : %c %s %s\n",
@ -70,8 +73,26 @@ func main() {
)
}
}
fmt.Printf("%+v\n", m)
var trees []*FileTree
trees = make([]*FileTree, 0)
for _, treeName := range m.Layers {
fmt.Printf("%s\n", treeName)
trees = append(trees, layerMap[treeName])
}
for ix := range trees {
if ix > 0 {
trees[0].Stack(trees[ix])
}
}
// fmt.Printf("%+v\n", m)
// fmt.Printf("%+v\n", layerMap)
fmt.Printf("%+v\n", trees)
// fmt.Printf("Tree 1 is: \n %+v\n", trees[1])
// fmt.Printf("Tree 2 is: \n %+v\n", trees[2])
// trees[1].Stack(trees[2])
// fmt.Printf("Tree 1 stacked with tree 2 is: \n %+v\n", trees[1])
// fmt.Printf("The whle stack is \n %+v \n", trees[0])
}
func getFileList(parentReader *tar.Reader, h *tar.Header) []FileChangeInfo {
@ -136,6 +157,7 @@ func makeEntry(r *tar.Reader, h *tar.Header, path string) FileChangeInfo {
path: path,
typeflag: h.Typeflag,
md5sum: hash,
diffType: Unchanged,
}
}
@ -145,6 +167,7 @@ type FileChangeInfo struct {
path string
typeflag byte
md5sum [16]byte
diffType DiffType
}
type Manifest struct {

16
tree.go
View File

@ -1,10 +1,10 @@
package main
import (
"errors"
"fmt"
"sort"
"strings"
"fmt"
"errors"
)
const (
@ -18,7 +18,6 @@ const (
collapsedItem = "⊕ "
)
type FileTree struct {
root *Node
size int
@ -43,9 +42,12 @@ func NewTree() (tree *FileTree) {
return tree
}
func NewNode(parent *Node, name string, data interface{}) (node *Node) {
func NewNode(parent *Node, name string, data *FileChangeInfo) (node *Node) {
node = new(Node)
node.name = name
if data == nil {
data = &FileChangeInfo{}
}
node.data = data
node.children = make(map[string]*Node)
node.parent = parent
@ -57,7 +59,7 @@ func (tree *FileTree) Root() *Node {
return tree.root
}
func (node *Node) AddChild(name string, data interface{}) (child *Node) {
func (node *Node) AddChild(name string, data *FileChangeInfo) (child *Node) {
child = NewNode(node, name, data)
if node.children[name] != nil {
// tree node already exists, replace the payload, keep the children
@ -190,7 +192,7 @@ func (node *Node) Path() string {
return "/" + strings.Join(path, "/")
}
func (tree *FileTree) Stack(upper *FileTree) (error) {
func (tree *FileTree) Stack(upper *FileTree) error {
graft := func(node *Node) error {
if node.IsWhiteout() {
err := tree.RemovePath(node.Path())
@ -223,7 +225,7 @@ func (tree *FileTree) GetNode(path string) (*Node, error) {
return node, nil
}
func (tree *FileTree) AddPath(path string, data interface{}) (*Node, error) {
func (tree *FileTree) AddPath(path string, data *FileChangeInfo) (*Node, error) {
nodeNames := strings.Split(path, "/")
node := tree.Root()
for idx, name := range nodeNames {

161
tree_compare.go Normal file
View File

@ -0,0 +1,161 @@
package main
import (
"bytes"
"fmt"
)
func differ() func(int) int {
sum := 0
return func(x int) int {
sum += x
return sum
}
}
type DiffType int
// enum to show whether a file has changed
const (
Unchanged DiffType = iota
Changed
Added
Removed
)
func (d DiffType) String() string {
switch d {
case Unchanged:
return "Unchanged"
case Changed:
return "Changed"
case Added:
return "Added"
case Removed:
return "Removed"
default:
return fmt.Sprintf("%d", int(d))
}
}
func compareNodes(a *Node, b *Node) DiffType {
if a == nil && b == nil {
return Unchanged
}
// a is nil but not b
if a == nil && b != nil {
return Added
}
// b is nil but not a
if a != nil && b == nil {
return Removed
}
if b.IsWhiteout() {
return Removed
}
if a.name != b.name {
panic("comparing mismatched nodes")
}
// TODO: fails on nil
return getDiffType(a.data, b.data)
}
func getDiffType(a *FileChangeInfo, b *FileChangeInfo) DiffType {
if a == nil && b == nil {
return Unchanged
}
if a == nil || b == nil {
return Changed
}
if a.typeflag == b.typeflag {
if bytes.Compare(a.md5sum[:], b.md5sum[:]) == 0 {
return Unchanged
}
}
return Changed
}
func mergeDiffTypes(a DiffType, b DiffType) DiffType {
if a == b {
return a
}
return Changed
}
func (tree *FileTree) compareTo(upper *FileTree) error {
// TODO mark all as unchanged
markAllUnchanged := func(node *Node) error {
return node.AssignDiffType(Unchanged)
}
err := tree.Visit(markAllUnchanged)
if err != nil {
return err
}
graft := func(node *Node) error {
if node.IsWhiteout() {
err := tree.MarkRemoved(node.Path())
if err != nil {
return fmt.Errorf("Cannot remove node %s: %v", node.Path(), err.Error())
}
} else {
existingNode, _ := tree.GetNode(node.Path())
if existingNode == nil {
newNode, err := tree.AddPath(node.Path(), node.data)
fmt.Printf("added new node at %s\n", newNode.Path())
if err != nil {
return fmt.Errorf("Cannot add new node %s: %v", node.Path(), err.Error())
}
newNode.AssignDiffType(Added)
} else {
diffType := compareNodes(existingNode, node)
fmt.Printf("found existing node at %s\n", existingNode.Path())
existingNode.DiffTypeFromChildren(diffType)
}
}
return nil
}
return upper.Visit(graft)
}
// THE DIFF_TYPE OF A NODE IS ALWAYS THE DIFF_TYPE OF ITS ATTRIBUTES AND ITS CONTENTS
// THE CONTENTS ARE THE BYTES OF A FILE OR THE CHILDREN OF A DIRECTORY
func (tree *FileTree) MarkRemoved(path string) error {
node, err := tree.GetNode(path)
if err != nil {
return err
}
return node.AssignDiffType(Removed)
}
func (node *Node) IsLeaf() bool {
return len(node.children) == 0
}
func (node *Node) DiffTypeFromChildren(diffType DiffType) error {
if node.IsLeaf() {
node.AssignDiffType(diffType)
return nil
}
myDiffType := diffType
for _, v := range node.children {
vData := v.data
myDiffType = mergeDiffTypes(myDiffType, vData.diffType)
}
node.AssignDiffType(myDiffType)
return nil
}
func (node *Node) AssignDiffType(diffType DiffType) error {
if node.Path() == "/" {
return nil
}
node.data.diffType = diffType
return nil
}

190
tree_compare_test.go Normal file
View File

@ -0,0 +1,190 @@
package main
import (
"fmt"
"testing"
)
func TestCompareWithNoChanges(t *testing.T) {
lowerTree := NewTree()
upperTree := NewTree()
paths := [...]string{"/etc", "/etc/sudoers", "/etc/hosts", "/usr/bin", "/usr/bin/bash", "/usr"}
for _, value := range paths {
fakeData := FileChangeInfo{
path: value,
typeflag: 1,
md5sum: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
diffType: Unchanged,
}
lowerTree.AddPath(value, &fakeData)
upperTree.AddPath(value, &fakeData)
}
lowerTree.compareTo(upperTree)
asserter := func(n *Node) error {
if n.Path() == "/" {
return nil
}
if n.data == nil {
t.Errorf("Expected *FileChangeInfo but got nil")
return fmt.Errorf("expected *FileChangeInfo but got nil")
}
if (n.data.diffType) != Unchanged {
t.Errorf("Expecting node at %s to have DiffType unchanged, but had %v", n.Path(), n.data.diffType)
}
return nil
}
err := lowerTree.Visit(asserter)
if err != nil {
t.Error(err)
}
}
func TestCompareWithAdds(t *testing.T) {
lowerTree := NewTree()
upperTree := NewTree()
lowerPaths := [...]string{"/etc", "/etc/sudoers", "/usr", "/etc/hosts", "/usr/bin"}
upperPaths := [...]string{"/etc", "/etc/sudoers", "/usr", "/etc/hosts", "/usr/bin", "/usr/bin/bash"}
for _, value := range lowerPaths {
fakeData := FileChangeInfo{
path: value,
typeflag: 1,
md5sum: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
diffType: Unchanged,
}
lowerTree.AddPath(value, &fakeData)
}
for _, value := range upperPaths {
fakeData := FileChangeInfo{
path: value,
typeflag: 1,
md5sum: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
diffType: Unchanged,
}
upperTree.AddPath(value, &fakeData)
}
lowerTree.compareTo(upperTree)
asserter := func(n *Node) error {
p := n.Path()
if p == "/" {
return nil
}
// Adding a file changes the folders it's in
if p == "/usr/bin/bash" {
return AssertDiffType(n, Added, t)
}
if p == "/usr/bin" {
return AssertDiffType(n, Changed, t)
}
if p == "/usr" {
return AssertDiffType(n, Changed, t)
}
return AssertDiffType(n, Unchanged, t)
}
err := lowerTree.Visit(asserter)
if err != nil {
t.Error(err)
}
}
func TestCompareWithChanges(t *testing.T) {
lowerTree := NewTree()
upperTree := NewTree()
lowerPaths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin"}
upperPaths := [...]string{"/etc", "/usr", "/etc/hosts", "/etc/sudoers", "/usr/bin"}
for _, value := range lowerPaths {
fakeData := FileChangeInfo{
path: value,
typeflag: 1,
md5sum: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
diffType: Unchanged,
}
lowerTree.AddPath(value, &fakeData)
}
for _, value := range upperPaths {
fakeData := FileChangeInfo{
path: value,
typeflag: 1,
md5sum: [16]byte{1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0},
diffType: Unchanged,
}
upperTree.AddPath(value, &fakeData)
}
lowerTree.compareTo(upperTree)
asserter := func(n *Node) error {
p := n.Path()
if p == "/" {
return nil
}
return AssertDiffType(n, Changed, t)
}
err := lowerTree.Visit(asserter)
if err != nil {
t.Error(err)
}
}
func TestAssignDiffType(t *testing.T) {
tree := NewTree()
tree.AddPath("/usr", BlankFileChangeInfo("/usr", Changed))
if tree.root.children["usr"].data.diffType != Changed {
t.Fail()
}
}
func TestMergeDiffTypes(t *testing.T) {
a := Unchanged
b := Unchanged
merged := mergeDiffTypes(a, b)
if merged != Unchanged {
t.Errorf("Expected Unchaged (0) but got %v", merged)
}
a = Changed
b = Unchanged
merged = mergeDiffTypes(a, b)
if merged != Changed {
t.Errorf("Expected Unchaged (0) but got %v", merged)
}
}
func TestDiffTypeFromChildren(t *testing.T) {
tree := NewTree()
tree.AddPath("/usr", BlankFileChangeInfo("/usr", Unchanged))
info1 := BlankFileChangeInfo("/usr/bin", Added)
tree.AddPath("/usr/bin", info1)
info2 := BlankFileChangeInfo("/usr/bin2", Removed)
tree.AddPath("/usr/bin2", info2)
tree.root.children["usr"].DiffTypeFromChildren(Unchanged)
if tree.root.children["usr"].data.diffType != Changed {
t.Errorf("Expected Changed but got %v", tree.root.children["usr"].data.diffType)
}
}
func AssertDiffType(node *Node, expectedDiffType DiffType, t *testing.T) error {
if node.data == nil {
t.Errorf("Expected *FileChangeInfo but got nil at path %s", node.Path())
return fmt.Errorf("expected *FileChangeInfo but got nil at path %s", node.Path())
}
if node.data.diffType != expectedDiffType {
t.Errorf("Expecting node at %s to have DiffType %v, but had %v", node.Path(), expectedDiffType, node.data.diffType)
return fmt.Errorf("Assertion failed")
}
return nil
}
func BlankFileChangeInfo(path string, diffType DiffType) (f *FileChangeInfo) {
result := FileChangeInfo{
path: path,
typeflag: 1,
md5sum: [16]byte{1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0},
diffType: diffType,
}
return &result
}

87
tree_test.go
View File

@ -6,14 +6,18 @@ func TestAddChild(t *testing.T) {
var expected, actual int
tree := NewTree()
one := tree.Root().AddChild("first node!", 1)
payload := FileChangeInfo{
path: "stufffffs",
}
one := tree.Root().AddChild("first node!", &payload)
two := tree.Root().AddChild("nil node!", nil)
tree.Root().AddChild("third node!", 3)
two.AddChild("forth, one level down...", 4)
two.AddChild("fifth, one level down...", 5)
two.AddChild("fifth, one level down...", 5)
tree.Root().AddChild("third node!", nil)
two.AddChild("forth, one level down...", nil)
two.AddChild("fifth, one level down...", nil)
two.AddChild("fifth, one level down...", nil)
expected, actual = 5, tree.size
if expected != actual {
@ -30,26 +34,31 @@ func TestAddChild(t *testing.T) {
t.Errorf("Expected 'twos' number of children to be %d got %d.", expected, actual)
}
if two.data != nil {
t.Errorf("Expected 'ones' payload to be nil got %d.", two.data)
expectedFC := &FileChangeInfo{
path: "stufffffs",
}
actualFC := one.data
if *expectedFC != *actualFC {
t.Errorf("Expected 'ones' payload to be %+v got %+v.", expectedFC, actualFC)
}
expected, actual = 1, one.data.(int)
if expected != actual {
t.Errorf("Expected 'ones' payload to be %d got %d.", expected, actual)
if *two.data != *new(FileChangeInfo) {
t.Errorf("Expected 'twos' payload to be nil got %d.", two.data)
}
}
func TestRemoveChild(t *testing.T) {
var expected, actual int
tree := NewTree()
tree.Root().AddChild("first", 1)
tree.Root().AddChild("first", nil)
two := tree.Root().AddChild("nil", nil)
tree.Root().AddChild("third", 3)
forth := two.AddChild("forth", 4)
two.AddChild("fifth", 5)
tree.Root().AddChild("third", nil)
forth := two.AddChild("forth", nil)
two.AddChild("fifth", nil)
forth.Remove()
@ -99,12 +108,12 @@ func TestPrintTree(t *testing.T) {
func TestAddPath(t *testing.T) {
tree := NewTree()
tree.AddPath("/etc/nginx/nginx.conf", 1)
tree.AddPath("/etc/nginx/public", 2)
tree.AddPath("/var/run/systemd", 3)
tree.AddPath("/var/run/bashful", 4)
tree.AddPath("/tmp", 5)
tree.AddPath("/tmp/nonsense", 6)
tree.AddPath("/etc/nginx/nginx.conf", nil)
tree.AddPath("/etc/nginx/public", nil)
tree.AddPath("/var/run/systemd", nil)
tree.AddPath("/var/run/bashful", nil)
tree.AddPath("/tmp", nil)
tree.AddPath("/tmp/nonsense", nil)
expected := `.
├── etc
@ -128,12 +137,12 @@ func TestAddPath(t *testing.T) {
func TestRemovePath(t *testing.T) {
tree := NewTree()
tree.AddPath("/etc/nginx/nginx.conf", 1)
tree.AddPath("/etc/nginx/public", 2)
tree.AddPath("/var/run/systemd", 3)
tree.AddPath("/var/run/bashful", 4)
tree.AddPath("/tmp", 5)
tree.AddPath("/tmp/nonsense", 6)
tree.AddPath("/etc/nginx/nginx.conf", nil)
tree.AddPath("/etc/nginx/public", nil)
tree.AddPath("/var/run/systemd", nil)
tree.AddPath("/var/run/bashful", nil)
tree.AddPath("/tmp", nil)
tree.AddPath("/tmp/nonsense", nil)
tree.RemovePath("/var/run/bashful")
tree.RemovePath("/tmp")
@ -168,8 +177,8 @@ func TestPath(t *testing.T) {
func TestIsWhiteout(t *testing.T) {
tree1 := NewTree()
p1, _ := tree1.AddPath("/etc/nginx/public1", 2)
p2, _ := tree1.AddPath("/etc/nginx/.wh.public2", 2)
p1, _ := tree1.AddPath("/etc/nginx/public1", nil)
p2, _ := tree1.AddPath("/etc/nginx/.wh.public2", nil)
if p1.IsWhiteout() != false {
t.Errorf("Expected path '%s' to **not** be a whiteout file", p1.name)
@ -183,21 +192,23 @@ func TestIsWhiteout(t *testing.T) {
func TestStack(t *testing.T) {
payloadKey := "/var/run/systemd"
payloadValue := 1263487
payloadValue := FileChangeInfo{
path: "yup",
}
tree1 := NewTree()
tree1.AddPath("/etc/nginx/public", 2)
tree1.AddPath(payloadKey, 3)
tree1.AddPath("/var/run/bashful", 4)
tree1.AddPath("/tmp", 5)
tree1.AddPath("/tmp/nonsense", 6)
tree1.AddPath("/etc/nginx/public", nil)
tree1.AddPath(payloadKey, nil)
tree1.AddPath("/var/run/bashful", nil)
tree1.AddPath("/tmp", nil)
tree1.AddPath("/tmp/nonsense", nil)
tree2 := NewTree()
// add new files
tree2.AddPath("/etc/nginx/nginx.conf", 1)
tree2.AddPath("/etc/nginx/nginx.conf", nil)
// modify current files
tree2.AddPath(payloadKey, payloadValue)
tree2.AddPath(payloadKey, &payloadValue)
// whiteout the following files
tree2.AddPath("/var/run/.wh.bashful", nil)
tree2.AddPath("/.wh.tmp", nil)
@ -223,8 +234,8 @@ func TestStack(t *testing.T) {
t.Errorf("Expected '%s' to still exist, but it doesn't", payloadKey)
}
if node.data != payloadValue {
t.Errorf("Expected '%s' value to be %d but got %d", payloadKey, payloadValue, node.data.(int))
if *node.data != payloadValue {
t.Errorf("Expected '%s' value to be %+v but got %+v", payloadKey, payloadValue, node.data)
}
actual := tree1.String()