diff --git a/go.mod b/go.mod index 6c248698e..2f1c36bd1 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,9 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.0 // indirect + github.com/DataDog/zstd v1.4.4 // indirect + github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 // indirect + github.com/asdine/storm v2.1.2+incompatible github.com/bradleyjkemp/cupaloy v2.3.0+incompatible github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 // indirect github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f @@ -15,7 +18,7 @@ require ( github.com/fatih/structs v1.0.0 // indirect github.com/go-chi/chi v4.0.3+incompatible github.com/go-chi/cors v1.0.0 - github.com/golang/snappy v0.0.0-20170215233205-553a64147049 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/wire v0.4.0 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629 github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a @@ -30,4 +33,8 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.4.0 // indirect github.com/syndtr/goleveldb v0.0.0-20170302031910-3c5717caf147 + github.com/vmihailenco/msgpack v4.0.1+incompatible // indirect + go.etcd.io/bbolt v1.3.3 // indirect + golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect + google.golang.org/appengine v1.6.5 // indirect ) diff --git a/go.sum b/go.sum index 7898119b5..6cf26003e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ github.com/BurntSushi/toml v0.3.0 h1:e1/Ivsx3Z0FVTV0NSOv/aVgbUWyQuzj7DDnFblkRvsY= github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= +github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM= +github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM= +github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q= +github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ= github.com/bradleyjkemp/cupaloy v2.3.0+incompatible h1:UafIjBvWQmS9i/xRg+CamMrnLTKNzo+bdmT/oH34c2Y= github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ= @@ -24,16 +30,16 @@ github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= -github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY= github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0= github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE= @@ -80,10 +86,16 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v0.0.0-20170302031910-3c5717caf147 h1:4YA7EV3fB/q1fi3RYWi26t91Zm6iHggaq8gJBRYC5Ms= github.com/syndtr/goleveldb v0.0.0-20170302031910-3c5717caf147/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU= +github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -91,13 +103,20 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/persistence/storm/artist_repository.go b/persistence/storm/artist_repository.go new file mode 100644 index 000000000..647dd7ca0 --- /dev/null +++ b/persistence/storm/artist_repository.go @@ -0,0 +1,47 @@ +package storm + +import ( + "github.com/cloudsonic/sonic-server/domain" +) + +// This is used to isolate Storm's struct tags from the domain, to keep it agnostic of persistence details +type _Artist struct { + ID string + Name string `storm:"index"` + AlbumCount int +} + +type artistRepository struct { + stormRepository +} + +func NewArtistRepository() domain.ArtistRepository { + r := &artistRepository{} + r.init(&_Artist{}) + return r +} + +func (r *artistRepository) Put(a *domain.Artist) error { + ta := _Artist(*a) + return Db().Save(&ta) +} + +func (r *artistRepository) Get(id string) (*domain.Artist, error) { + ta := &_Artist{} + + err := Db().One("ID", id, ta) + a := domain.Artist(*ta) + return &a, err +} + +func (r *artistRepository) PurgeInactive(active domain.Artists) ([]string, error) { + activeIDs := make([]string, len(active)) + for i, artist := range active { + activeIDs[i] = artist.ID + } + + return r.purgeInactive(activeIDs) +} + +var _ domain.ArtistRepository = (*artistRepository)(nil) +var _ = domain.Artist(_Artist{}) diff --git a/persistence/storm/artist_repository_test.go b/persistence/storm/artist_repository_test.go new file mode 100644 index 000000000..88800023b --- /dev/null +++ b/persistence/storm/artist_repository_test.go @@ -0,0 +1,41 @@ +package storm + +import ( + "github.com/cloudsonic/sonic-server/domain" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtistRepository", func() { + var repo domain.ArtistRepository + + BeforeEach(func() { + repo = NewArtistRepository() + }) + + It("saves and retrieves data", func() { + artist := &domain.Artist{ + ID: "1", + Name: "Saara Saara", + AlbumCount: 2, + } + Expect(repo.Put(artist)).To(BeNil()) + + Expect(repo.Get("1")).To(Equal(artist)) + }) + + It("purges inactive records", func() { + data := domain.Artists{ + {ID: "1", Name: "Saara Saara"}, + {ID: "2", Name: "Kraftwerk"}, + {ID: "3", Name: "The Beatles"}, + } + active := domain.Artists{ + {ID: "1"}, {ID: "3"}, + } + for _, a := range data { + repo.Put(&a) + } + Expect(repo.PurgeInactive(active)).To(Equal([]string{"2"})) + }) +}) diff --git a/persistence/storm/domain_tagging.go b/persistence/storm/domain_tagging.go new file mode 100644 index 000000000..ab38b95ab --- /dev/null +++ b/persistence/storm/domain_tagging.go @@ -0,0 +1,42 @@ +package storm + +import ( + "reflect" +) + +func tag(entity interface{}) interface{} { + st := reflect.TypeOf(entity).Elem() + var fs []reflect.StructField + for i := 0; i < st.NumField(); i++ { + f := st.Field(i) + f.Tag = mapTags(f.Tag) + fs = append(fs, f) + } + + st2 := reflect.StructOf(fs) + v := reflect.ValueOf(entity).Elem() + v2 := v.Convert(st2) + vp := reflect.New(st2) + vp.Elem().Set(reflect.ValueOf(v2.Interface())) + return vp.Interface() +} + +func mapTags(tags reflect.StructTag) reflect.StructTag { + if tags == `db:"index"` { + return `storm:"index"` + } + return tags +} + +func getTypeName(myVar interface{}) string { + if t := reflect.TypeOf(myVar); t.Kind() == reflect.Ptr { + return t.Elem().Name() + } else { + return t.Name() + } +} + +func getStructTag(instance interface{}, fieldName string) string { + field, _ := reflect.TypeOf(instance).Elem().FieldByName(fieldName) + return string(field.Tag) +} diff --git a/persistence/storm/domain_tagging_test.go b/persistence/storm/domain_tagging_test.go new file mode 100644 index 000000000..7617898a8 --- /dev/null +++ b/persistence/storm/domain_tagging_test.go @@ -0,0 +1,33 @@ +package storm + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +type Empty struct { + ID string + Something int +} + +type User struct { + ID string + Name string `db:"index"` +} + +var _ = Describe("Domain Tagging", func() { + + It("does not change a struct that does not have any tag", func() { + empty := &Empty{} + tagged := tag(empty) + Expect(getStructTag(tagged, "ID")).To(BeEmpty()) + Expect(getStructTag(tagged, "Something")).To(BeEmpty()) + }) + + It("adds index to indexed fields", func() { + user := &User{} + tagged := tag(user) + Expect(getStructTag(tagged, "ID")).To(BeEmpty()) + Expect(getStructTag(tagged, "Name")).To(Equal(`storm:"index"`)) + }) +}) diff --git a/persistence/storm/property_repository.go b/persistence/storm/property_repository.go new file mode 100644 index 000000000..d0186061b --- /dev/null +++ b/persistence/storm/property_repository.go @@ -0,0 +1,31 @@ +package storm + +import ( + "github.com/cloudsonic/sonic-server/domain" +) + +const propertyBucket = "Property" + +type propertyRepository struct { +} + +func NewPropertyRepository() domain.PropertyRepository { + r := &propertyRepository{} + return r +} + +func (r *propertyRepository) Put(id string, value string) error { + return Db().Set(propertyBucket, id, value) +} + +func (r *propertyRepository) Get(id string) (string, error) { + var value string + err := Db().Get(propertyBucket, id, &value) + return value, err +} + +func (r *propertyRepository) DefaultGet(id string, defaultValue string) (string, error) { + return defaultValue, nil +} + +var _ domain.PropertyRepository = (*propertyRepository)(nil) diff --git a/persistence/storm/storm.go b/persistence/storm/storm.go new file mode 100644 index 000000000..3c9fcf32e --- /dev/null +++ b/persistence/storm/storm.go @@ -0,0 +1,23 @@ +package storm + +import ( + "sync" + + "github.com/asdine/storm" +) + +var ( + _dbInstance *storm.DB + once sync.Once +) + +func Db() *storm.DB { + once.Do(func() { + instance, err := storm.Open("./storm.db") + if err != nil { + panic(err) + } + _dbInstance = instance + }) + return _dbInstance +} diff --git a/persistence/storm/storm_repository.go b/persistence/storm/storm_repository.go new file mode 100644 index 000000000..0b1cab68b --- /dev/null +++ b/persistence/storm/storm_repository.go @@ -0,0 +1,55 @@ +package storm + +import ( + "reflect" + + "github.com/asdine/storm" + "github.com/asdine/storm/q" +) + +type stormRepository struct { + bucket interface{} +} + +func (r *stormRepository) init(entity interface{}) { + r.bucket = entity + if err := Db().Init(r.bucket); err != nil { + panic(err) + } + if err := Db().ReIndex(r.bucket); err != nil { + panic(err) + } +} + +func (r *stormRepository) CountAll() (int64, error) { + c, err := Db().Count(r.bucket) + return int64(c), err +} + +func (r *stormRepository) Exists(id string) (bool, error) { + err := Db().One("ID", id, r.bucket) + if err != nil { + return false, err + } + return err != storm.ErrNotFound, nil +} + +func (r *stormRepository) purgeInactive(ids []string) (deleted []string, err error) { + query := Db().Select(q.Not(q.In("ID", ids))) + + err = query.Each(r.bucket, func(record interface{}) error { + v := reflect.ValueOf(record).Elem() + id := v.FieldByName("ID").String() + deleted = append(deleted, id) + return nil + }) + if err != nil { + return nil, err + } + + err = query.Delete(r.bucket) + if err != nil { + return nil, err + } + return deleted, nil +} diff --git a/persistence/storm/storm_suite_test.go b/persistence/storm/storm_suite_test.go new file mode 100644 index 000000000..61c1f1e4f --- /dev/null +++ b/persistence/storm/storm_suite_test.go @@ -0,0 +1,14 @@ +package storm + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStormPersistence(t *testing.T) { + //log.SetLevel(log.LevelCritical) + RegisterFailHandler(Fail) + RunSpecs(t, "Storm Persistence Suite") +} diff --git a/persistence/storm/wire_providers.go b/persistence/storm/wire_providers.go new file mode 100644 index 000000000..30bfaf499 --- /dev/null +++ b/persistence/storm/wire_providers.go @@ -0,0 +1,8 @@ +package storm + +import "github.com/google/wire" + +var Set = wire.NewSet( + NewPropertyRepository, + NewArtistRepository, +)