From f64f21897054cd4297d370663f15dc2788b948ff Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Fri, 27 Sep 2019 19:47:27 -0400 Subject: [PATCH 01/10] Code to convert Table to KV store --- tablestore/test.go | 119 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tablestore/test.go diff --git a/tablestore/test.go b/tablestore/test.go new file mode 100644 index 0000000..0cc89f6 --- /dev/null +++ b/tablestore/test.go @@ -0,0 +1,119 @@ +package main + +import ( + "fmt" + "strings" + "github.com/acharapko/fleetdb/kv_store" + //"github.com/acharapko/fleetdb/ids" +) +/* +CREATE TABLE crossfit_gyms ( + country_code text, + state_province text, + city text, + gym_name text, + PRIMARY KEY (country_code, state_province) +); +*/ + + +type FleetDBType uint8 + +const ( + INT FleetDBType = iota + FLOAT + TEXT +) + +type KVItem struct { + Key []byte + Value []byte +} + +type FleetDbColumnSpec struct { + colname string + coltype FleetDBType + isPartition bool + isClustering bool +} +//m := make(map[string]int) +func main(){ + createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" + fmt.Println(createCommand) + tableMap := make(map[string][]FleetDbColumnSpec) + myschema,tableName := createSchema(createCommand) + tableMap[tableName] = myschema + insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; + rowData, myTableName := decodeInsertCommand(insertCommand) + //kvData := TranslateToKV(tableMap[myTableName], rowData) + TranslateToKV(tableMap[myTableName], rowData) + fmt.Println("before new store") + kv_store.NewStore() + fmt.Println("After new store") + /*for _, item := range kvItem{ + cmd := kv_store.Command{tableName, item.Key, item.Value, "", 0, kv_store.PUT } + //var id = flag.String("id", "1.1", "ID in format of Zone.Node. Default 1.1") + id := ids.NewID(5,5) + _, err := store.Execute(cmd, id) + if err != nil { + fmt.Println("Put Sucessful") + }else { + fmt.Println("Put Unsucessful") + } + } + */ + +// success := writeToDB(myTableName, ksvData) +// if sucess { +// fmt.Println("Write Successful") +// } +// returnData = readFromDB(myTableName) +// isEqual := checkEqality(returnData, kvData) +// if isEqual { +// fmt.Println("Write and Read Happning correctly") +// } +} + +func createSchema(query string) ([]FleetDbColumnSpec,string) { + schema := make([]FleetDbColumnSpec, 4) + schema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} + schema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} + schema[2] = FleetDbColumnSpec{"city", TEXT, false, false} + schema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + return schema,"crossfit_gyms" +} + +func decodeInsertCommand(query string)([][]string , string){ + val := [][]string{{"country_code", "US"},{"state_province", "NY"},{"city", "Buffalo"},{"gym_name", "University Avenue"}} + return val, "crossfit_gyms" +} + +func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]string) []KVItem{ + //build common string + var sbPKey strings.Builder + var sbCKey strings.Builder + for i, colSpec := range columnSpecs { + if colSpec.isPartition{ + fmt.Println(values[i][1]) + sbPKey.WriteString(values[i][1]) + } + if colSpec.isClustering{ + fmt.Println(values[i][1]) + sbCKey.WriteString(values[i][1]) + } + } + kvItems := make([]KVItem, 2) + index := 0 + prefix := sbPKey.String() + "/"+ sbCKey.String() + for i, colSpec := range columnSpecs { + if !colSpec.isPartition && !colSpec.isClustering{ + key := prefix + "/" + values[i][0] + val := values[i][1] + fmt.Println(key + " = " + val) + kvItems[index] = KVItem{[]byte(key), []byte(val)} + index = index + 1 + } + } + //sbPKey.WriteString(str) + return kvItems; +} From 4adb88d1b5af027159e34528331bc6037ca513d7 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Tue, 1 Oct 2019 11:25:47 -0400 Subject: [PATCH 02/10] changed Translate method to Accept byte[][] --- tablestore/test.go | 62 ++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/tablestore/test.go b/tablestore/test.go index 0cc89f6..c4ae4c7 100644 --- a/tablestore/test.go +++ b/tablestore/test.go @@ -2,9 +2,9 @@ package main import ( "fmt" - "strings" - "github.com/acharapko/fleetdb/kv_store" - //"github.com/acharapko/fleetdb/ids" + //"strings" + //"github.com/darshannevgi/fleetdb/kv_store" + //"github.com/darshannevgi/fleetdb/tablestore" ) /* CREATE TABLE crossfit_gyms ( @@ -16,7 +16,6 @@ CREATE TABLE crossfit_gyms ( ); */ - type FleetDBType uint8 const ( @@ -25,17 +24,18 @@ const ( TEXT ) -type KVItem struct { - Key []byte - Value []byte -} - type FleetDbColumnSpec struct { colname string coltype FleetDBType isPartition bool isClustering bool } + +type KVItem struct { + Key []byte + Value []byte +} + //m := make(map[string]int) func main(){ createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" @@ -47,9 +47,9 @@ func main(){ rowData, myTableName := decodeInsertCommand(insertCommand) //kvData := TranslateToKV(tableMap[myTableName], rowData) TranslateToKV(tableMap[myTableName], rowData) - fmt.Println("before new store") - kv_store.NewStore() - fmt.Println("After new store") + //fmt.Println("before new store") + //kv_store.NewStore() + //fmt.Println("After new store") /*for _, item := range kvItem{ cmd := kv_store.Command{tableName, item.Key, item.Value, "", 0, kv_store.PUT } //var id = flag.String("id", "1.1", "ID in format of Zone.Node. Default 1.1") @@ -83,37 +83,39 @@ func createSchema(query string) ([]FleetDbColumnSpec,string) { return schema,"crossfit_gyms" } -func decodeInsertCommand(query string)([][]string , string){ - val := [][]string{{"country_code", "US"},{"state_province", "NY"},{"city", "Buffalo"},{"gym_name", "University Avenue"}} +func decodeInsertCommand(query string)([][]byte , string){ + val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} return val, "crossfit_gyms" } - -func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]string) []KVItem{ - //build common string - var sbPKey strings.Builder - var sbCKey strings.Builder +/* +Each values[0] represents byte[] data value of first column +if first column is country_code then values[0] is byte representation of 'US' = byte[]{85,83} +*/ +func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ + var pKey []byte + var cKey []byte for i, colSpec := range columnSpecs { if colSpec.isPartition{ - fmt.Println(values[i][1]) - sbPKey.WriteString(values[i][1]) + pKey = append(pKey, values[i]...) } if colSpec.isClustering{ - fmt.Println(values[i][1]) - sbCKey.WriteString(values[i][1]) + cKey = append(cKey, values[i]...) } } - kvItems := make([]KVItem, 2) + kvItems := make([]KVItem, 100) index := 0 - prefix := sbPKey.String() + "/"+ sbCKey.String() + pKey = append(pKey,"/"...) + prefix := append(pKey,cKey...) + prefix = append(prefix,"/"...) for i, colSpec := range columnSpecs { if !colSpec.isPartition && !colSpec.isClustering{ - key := prefix + "/" + values[i][0] - val := values[i][1] - fmt.Println(key + " = " + val) - kvItems[index] = KVItem{[]byte(key), []byte(val)} + key := append(prefix,colSpec.colname...) + val := values[i] + fmt.Println("Key is =" + string(key)) + fmt.Println("Value is =" + string(val)) + kvItems[index] = KVItem{key, val} index = index + 1 } } - //sbPKey.WriteString(str) return kvItems; } From 1b5b2dabc93731189eb82bae31275c1f36eb44b8 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Tue, 1 Oct 2019 12:14:35 -0400 Subject: [PATCH 03/10] File rearrangement and unit test file added --- tablestore/kv_item.go | 9 +++ tablestore/table_schema.go | 19 +++++ tablestore/test.go | 121 ------------------------------- tablestore/translatetoKV_test.go | 36 +++++++++ tablestore/utility.go | 40 ++++++++++ 5 files changed, 104 insertions(+), 121 deletions(-) create mode 100644 tablestore/kv_item.go create mode 100644 tablestore/table_schema.go delete mode 100644 tablestore/test.go create mode 100644 tablestore/translatetoKV_test.go create mode 100644 tablestore/utility.go diff --git a/tablestore/kv_item.go b/tablestore/kv_item.go new file mode 100644 index 0000000..ca6eb40 --- /dev/null +++ b/tablestore/kv_item.go @@ -0,0 +1,9 @@ +package tablestore + +import ( + +) +type KVItem struct { + Key []byte + Value []byte +} \ No newline at end of file diff --git a/tablestore/table_schema.go b/tablestore/table_schema.go new file mode 100644 index 0000000..00483b9 --- /dev/null +++ b/tablestore/table_schema.go @@ -0,0 +1,19 @@ +package tablestore + +import ( + +) +type FleetDBType uint8 + +const ( + INT FleetDBType = iota + FLOAT + TEXT +) + +type FleetDbColumnSpec struct { + colname string + coltype FleetDBType + isPartition bool + isClustering bool +} diff --git a/tablestore/test.go b/tablestore/test.go deleted file mode 100644 index c4ae4c7..0000000 --- a/tablestore/test.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "fmt" - //"strings" - //"github.com/darshannevgi/fleetdb/kv_store" - //"github.com/darshannevgi/fleetdb/tablestore" -) -/* -CREATE TABLE crossfit_gyms ( - country_code text, - state_province text, - city text, - gym_name text, - PRIMARY KEY (country_code, state_province) -); -*/ - -type FleetDBType uint8 - -const ( - INT FleetDBType = iota - FLOAT - TEXT -) - -type FleetDbColumnSpec struct { - colname string - coltype FleetDBType - isPartition bool - isClustering bool -} - -type KVItem struct { - Key []byte - Value []byte -} - -//m := make(map[string]int) -func main(){ - createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" - fmt.Println(createCommand) - tableMap := make(map[string][]FleetDbColumnSpec) - myschema,tableName := createSchema(createCommand) - tableMap[tableName] = myschema - insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; - rowData, myTableName := decodeInsertCommand(insertCommand) - //kvData := TranslateToKV(tableMap[myTableName], rowData) - TranslateToKV(tableMap[myTableName], rowData) - //fmt.Println("before new store") - //kv_store.NewStore() - //fmt.Println("After new store") - /*for _, item := range kvItem{ - cmd := kv_store.Command{tableName, item.Key, item.Value, "", 0, kv_store.PUT } - //var id = flag.String("id", "1.1", "ID in format of Zone.Node. Default 1.1") - id := ids.NewID(5,5) - _, err := store.Execute(cmd, id) - if err != nil { - fmt.Println("Put Sucessful") - }else { - fmt.Println("Put Unsucessful") - } - } - */ - -// success := writeToDB(myTableName, ksvData) -// if sucess { -// fmt.Println("Write Successful") -// } -// returnData = readFromDB(myTableName) -// isEqual := checkEqality(returnData, kvData) -// if isEqual { -// fmt.Println("Write and Read Happning correctly") -// } -} - -func createSchema(query string) ([]FleetDbColumnSpec,string) { - schema := make([]FleetDbColumnSpec, 4) - schema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} - schema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} - schema[2] = FleetDbColumnSpec{"city", TEXT, false, false} - schema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} - return schema,"crossfit_gyms" -} - -func decodeInsertCommand(query string)([][]byte , string){ - val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} - return val, "crossfit_gyms" -} -/* -Each values[0] represents byte[] data value of first column -if first column is country_code then values[0] is byte representation of 'US' = byte[]{85,83} -*/ -func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ - var pKey []byte - var cKey []byte - for i, colSpec := range columnSpecs { - if colSpec.isPartition{ - pKey = append(pKey, values[i]...) - } - if colSpec.isClustering{ - cKey = append(cKey, values[i]...) - } - } - kvItems := make([]KVItem, 100) - index := 0 - pKey = append(pKey,"/"...) - prefix := append(pKey,cKey...) - prefix = append(prefix,"/"...) - for i, colSpec := range columnSpecs { - if !colSpec.isPartition && !colSpec.isClustering{ - key := append(prefix,colSpec.colname...) - val := values[i] - fmt.Println("Key is =" + string(key)) - fmt.Println("Value is =" + string(val)) - kvItems[index] = KVItem{key, val} - index = index + 1 - } - } - return kvItems; -} diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go new file mode 100644 index 0000000..40471ae --- /dev/null +++ b/tablestore/translatetoKV_test.go @@ -0,0 +1,36 @@ +package tablestore + +import ( + "testing" + "fmt" +) + +func TestTranslateToKV(t *testing.T) { + + + //Test for Valid Arguments + createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" + fmt.Println(createCommand) + tableMap := make(map[string][]FleetDbColumnSpec) + myschema,tableName := createSchema(createCommand) + tableMap[tableName] = myschema + insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; + rowData, myTableName := decodeInsertCommand(insertCommand) + TranslateToKV(tableMap[myTableName], rowData) + + +} + +func createSchema(query string) ([]FleetDbColumnSpec,string) { + schema := make([]FleetDbColumnSpec, 4) + schema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} + schema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} + schema[2] = FleetDbColumnSpec{"city", TEXT, false, false} + schema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + return schema,"crossfit_gyms" +} + +func decodeInsertCommand(query string)([][]byte , string){ + val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} + return val, "crossfit_gyms" +} \ No newline at end of file diff --git a/tablestore/utility.go b/tablestore/utility.go new file mode 100644 index 0000000..23492b7 --- /dev/null +++ b/tablestore/utility.go @@ -0,0 +1,40 @@ +package tablestore + +import ( + "fmt" + //"strings" + //"github.com/darshannevgi/fleetdb/kv_store" + //"github.com/darshannevgi/fleetdb/tablestore" +) +/* +Each values[0] represents byte[] data value of first column +if first column is country_code then values[0] is byte representation of 'US' = byte[]{85,83} +*/ +func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ + var pKey []byte + var cKey []byte + for i, colSpec := range columnSpecs { + if colSpec.isPartition{ + pKey = append(pKey, values[i]...) + } + if colSpec.isClustering{ + cKey = append(cKey, values[i]...) + } + } + kvItems := make([]KVItem, 100) + index := 0 + pKey = append(pKey,"/"...) + prefix := append(pKey,cKey...) + prefix = append(prefix,"/"...) + for i, colSpec := range columnSpecs { + if !colSpec.isPartition && !colSpec.isClustering{ + key := append(prefix,colSpec.colname...) + val := values[i] + fmt.Println("Key is =" + string(key)) + fmt.Println("Value is =" + string(val)) + kvItems[index] = KVItem{key, val} + index = index + 1 + } + } + return kvItems; +} From 79a4bfce0169f58ecc93dadb4a40a96aab1ae321 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Tue, 1 Oct 2019 12:59:22 -0400 Subject: [PATCH 04/10] Added test cases to check diffrent Partition and Cluster Key combo --- tablestore/translatetoKV_test.go | 79 +++++++++++++++++++++++++++----- tablestore/utility.go | 11 +++-- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index 40471ae..f71a562 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -2,26 +2,85 @@ package tablestore import ( "testing" - "fmt" + // "fmt" ) func TestTranslateToKV(t *testing.T) { - //Test for Valid Arguments - createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" - fmt.Println(createCommand) + + //createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" + //fmt.Println(createCommand) + //myschema,tableName := createSchemaOnePOneC(createCommand) tableMap := make(map[string][]FleetDbColumnSpec) - myschema,tableName := createSchema(createCommand) + tableName := "crossfit_gyms" + + + //Test With One Partition Key and One Clustering Key + myschema := make([]FleetDbColumnSpec, 4) + myschema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} + myschema[2] = FleetDbColumnSpec{"city", TEXT, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + tableMap[tableName] = myschema insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; rowData, myTableName := decodeInsertCommand(insertCommand) - TranslateToKV(tableMap[myTableName], rowData) + res := TranslateToKV(tableMap[myTableName], rowData) + if string(res[0].Key) != "US/NY/city" { + t.Errorf("TranslateToKV() failed, expected %v, got %v", "US/NY/city" , string(res[0].Key)) + } + if string(res[0].Value) != "Buffalo" { + t.Errorf("TranslateToKV() failed, expected %v, got %v", "Buffalo" , string(res[0].Value)) + } + if string(res[1].Key) != "US/NY/gym_name" { + t.Errorf("TranslateToKV() failed, expected %v, got %v", "US/NY/gym_name" , string(res[1].Key)) + } + if string(res[1].Value) != "University Avenue" { + t.Errorf("TranslateToKV() failed, expected %v, got %v", "University Avenue" , string(res[1].Value)) + } + //fmt.Println(string(res[0].Key)); + + + //Test With One Partition Key and Zero Clustering Key + myschema = make([]FleetDbColumnSpec, 4) + myschema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", TEXT, false, false} + myschema[2] = FleetDbColumnSpec{"city", TEXT, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + + tableMap[tableName] = myschema + insertCommand = "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; + rowData, myTableName = decodeInsertCommand(insertCommand) + res = TranslateToKV(tableMap[myTableName], rowData) + if string(res[0].Key) != "US/state_province" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/state_province" , string(res[0].Key)) + } + if string(res[0].Value) != "NY" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "NY" , string(res[0].Value)) + } + if string(res[1].Key) != "US/city" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/city" , string(res[1].Key)) + } + if string(res[1].Value) != "Buffalo" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "Buffalo" , string(res[1].Value)) + } + if string(res[2].Key) != "US/gym_name" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/gym_name" , string(res[2].Key)) + } + if string(res[2].Value) != "University Avenue" { + t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "University Avenue" , string(res[2].Value)) + } } -func createSchema(query string) ([]FleetDbColumnSpec,string) { +func decodeInsertCommand(query string)([][]byte , string){ + val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} + return val, "crossfit_gyms" +} + +/*func createSchema(query string) ([]FleetDbColumnSpec,string) { schema := make([]FleetDbColumnSpec, 4) schema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} schema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} @@ -29,8 +88,4 @@ func createSchema(query string) ([]FleetDbColumnSpec,string) { schema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} return schema,"crossfit_gyms" } - -func decodeInsertCommand(query string)([][]byte , string){ - val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} - return val, "crossfit_gyms" -} \ No newline at end of file +*/ diff --git a/tablestore/utility.go b/tablestore/utility.go index 23492b7..c2e5f6f 100644 --- a/tablestore/utility.go +++ b/tablestore/utility.go @@ -23,9 +23,14 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ } kvItems := make([]KVItem, 100) index := 0 - pKey = append(pKey,"/"...) - prefix := append(pKey,cKey...) - prefix = append(prefix,"/"...) + var prefix []byte + prefix = pKey + prefix = append(prefix,"/"...) + if len(cKey) > 0{ + prefix = append(prefix,cKey...) + prefix = append(prefix,"/"...) + } + for i, colSpec := range columnSpecs { if !colSpec.isPartition && !colSpec.isClustering{ key := append(prefix,colSpec.colname...) From b20c4c7bcb7cccbd2fe5292b462afc87aa1bc83f Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Thu, 3 Oct 2019 17:12:23 -0400 Subject: [PATCH 05/10] Added Interface FleetDBType and Int implementation --- tablestore/FleetDBType.go | 33 ++++++++++++++++++++++++++++++++ tablestore/fleetDBType_test.go | 16 ++++++++++++++++ tablestore/table_schema.go | 7 ------- tablestore/translatetoKV_test.go | 31 ++++++++++++++++-------------- tablestore/utility.go | 12 ++++++++---- 5 files changed, 74 insertions(+), 25 deletions(-) create mode 100644 tablestore/FleetDBType.go create mode 100644 tablestore/fleetDBType_test.go diff --git a/tablestore/FleetDBType.go b/tablestore/FleetDBType.go new file mode 100644 index 0000000..71abc5c --- /dev/null +++ b/tablestore/FleetDBType.go @@ -0,0 +1,33 @@ +package tablestore + +import ( + "encoding/binary" + "fmt" +) +type FleetDBType interface{ + Serialize() []byte // this translates to []bytes + Deserialize([]byte, int) *FleetDBType //new type + //FromString(string) *FleetDBType //new from string +} +type Int struct{ + val uint32 +} + +func (i Int) String() string { + return fmt.Sprintf("Integer Value = %v", i.val) +} + +func (i Int) Serialize() []byte{ + bs := make([]byte, 4) + binary.LittleEndian.PutUint32(bs, i.val) + return bs + //fmt.Println(bs) +} + +func (i Int) Deserialize(values []byte, offset int) *FleetDBType{ + var myInt FleetDBType + data := binary.LittleEndian.Uint32(values) + myInt = Int{data} + return &myInt + //fmt.Println(myInt) +} \ No newline at end of file diff --git a/tablestore/fleetDBType_test.go b/tablestore/fleetDBType_test.go new file mode 100644 index 0000000..ba4fbee --- /dev/null +++ b/tablestore/fleetDBType_test.go @@ -0,0 +1,16 @@ +package tablestore + +import ( + "testing" + "fmt" +) + +func TestSerialize(t *testing.T) { + myInt := Int{200000} + byteSlice := myInt.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + newInt := myInt.Deserialize(byteSlice, 0) + fmt.Println((*newInt)) +} + diff --git a/tablestore/table_schema.go b/tablestore/table_schema.go index 00483b9..cc3cc1b 100644 --- a/tablestore/table_schema.go +++ b/tablestore/table_schema.go @@ -3,13 +3,6 @@ package tablestore import ( ) -type FleetDBType uint8 - -const ( - INT FleetDBType = iota - FLOAT - TEXT -) type FleetDbColumnSpec struct { colname string diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index f71a562..ff499de 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -9,19 +9,19 @@ func TestTranslateToKV(t *testing.T) { - //createCommand := "CREATE TABLE crossfit_gyms (country_code text,state_province text,city text,gym_name text,PRIMARY KEY (country_code, state_province));" + //createCommand := "CREATE TABLE crossfit_gyms (country_code string,state_province string,city string,gym_name string,PRIMARY KEY (country_code, state_province));" //fmt.Println(createCommand) //myschema,tableName := createSchemaOnePOneC(createCommand) tableMap := make(map[string][]FleetDbColumnSpec) tableName := "crossfit_gyms" - + var myInt Int //Test With One Partition Key and One Clustering Key myschema := make([]FleetDbColumnSpec, 4) - myschema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} - myschema[2] = FleetDbColumnSpec{"city", TEXT, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, true} + myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} tableMap[tableName] = myschema insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; @@ -44,10 +44,10 @@ func TestTranslateToKV(t *testing.T) { //Test With One Partition Key and Zero Clustering Key myschema = make([]FleetDbColumnSpec, 4) - myschema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", TEXT, false, false} - myschema[2] = FleetDbColumnSpec{"city", TEXT, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, false} + myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} tableMap[tableName] = myschema insertCommand = "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; @@ -72,6 +72,9 @@ func TestTranslateToKV(t *testing.T) { t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "University Avenue" , string(res[2].Value)) } + + + // } @@ -82,10 +85,10 @@ func decodeInsertCommand(query string)([][]byte , string){ /*func createSchema(query string) ([]FleetDbColumnSpec,string) { schema := make([]FleetDbColumnSpec, 4) - schema[0] = FleetDbColumnSpec{"country_code", TEXT, true, false} - schema[1] = FleetDbColumnSpec{"state_province", TEXT, false, true} - schema[2] = FleetDbColumnSpec{"city", TEXT, false, false} - schema[3] = FleetDbColumnSpec{"gym_name", TEXT, false, false} + schema[0] = FleetDbColumnSpec{"country_code", string, true, false} + schema[1] = FleetDbColumnSpec{"state_province", string, false, true} + schema[2] = FleetDbColumnSpec{"city", string, false, false} + schema[3] = FleetDbColumnSpec{"gym_name", string, false, false} return schema,"crossfit_gyms" } */ diff --git a/tablestore/utility.go b/tablestore/utility.go index c2e5f6f..38bdac2 100644 --- a/tablestore/utility.go +++ b/tablestore/utility.go @@ -1,7 +1,7 @@ package tablestore import ( - "fmt" + //"fmt" //"strings" //"github.com/darshannevgi/fleetdb/kv_store" //"github.com/darshannevgi/fleetdb/tablestore" @@ -21,7 +21,11 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ cKey = append(cKey, values[i]...) } } - kvItems := make([]KVItem, 100) + //Handle composite key and only pK and cK case + //In case of composite key , Key is made in order of partition key specified in create table command + //if PRIMARY KEY ((country_code, state_province, city) then key will be US/NY/Buffalo + + kvItems := make([]KVItem, len(values)) index := 0 var prefix []byte prefix = pKey @@ -35,8 +39,8 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ if !colSpec.isPartition && !colSpec.isClustering{ key := append(prefix,colSpec.colname...) val := values[i] - fmt.Println("Key is =" + string(key)) - fmt.Println("Value is =" + string(val)) + //fmt.Println("Key is =" + string(key)) + //fmt.Println("Value is =" + string(val)) kvItems[index] = KVItem{key, val} index = index + 1 } From 79373d9061091011d01871de5ea4fdd3f21a5518 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Fri, 4 Oct 2019 22:07:03 -0400 Subject: [PATCH 06/10] Adding FleetDBType interface Implementation for string, long and double --- tablestore/FleetDBType.go | 133 +++++++++++++++++++++++++++++++-- tablestore/fleetDBType_test.go | 28 +++++++ 2 files changed, 153 insertions(+), 8 deletions(-) diff --git a/tablestore/FleetDBType.go b/tablestore/FleetDBType.go index 71abc5c..c703a93 100644 --- a/tablestore/FleetDBType.go +++ b/tablestore/FleetDBType.go @@ -3,6 +3,7 @@ package tablestore import ( "encoding/binary" "fmt" + "bytes" ) type FleetDBType interface{ Serialize() []byte // this translates to []bytes @@ -10,7 +11,23 @@ type FleetDBType interface{ //FromString(string) *FleetDBType //new from string } type Int struct{ - val uint32 + val int32 +} + +type Float struct{ + val float32 +} + +type Long struct{ + val int64 +} + +type Double struct{ + val float64 +} + +type String struct{ + val string } func (i Int) String() string { @@ -18,16 +35,116 @@ func (i Int) String() string { } func (i Int) Serialize() []byte{ - bs := make([]byte, 4) - binary.LittleEndian.PutUint32(bs, i.val) - return bs - //fmt.Println(bs) + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, i.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() } func (i Int) Deserialize(values []byte, offset int) *FleetDBType{ var myInt FleetDBType - data := binary.LittleEndian.Uint32(values) + var data int32 + buf := bytes.NewReader(values) + err := binary.Read(buf, binary.LittleEndian, &data) + if err != nil { + fmt.Println("binary.Read failed:", err) + } myInt = Int{data} return &myInt - //fmt.Println(myInt) -} \ No newline at end of file +} + +func (f Float) String() string{ + return fmt.Sprintf("Float value = %v", f.val) +} + +func (f Float) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, f.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (f Float) Deserialize(values []byte, offset int) *FleetDBType { + var myFloat FleetDBType + var data float32 + buf := bytes.NewReader(values) + err := binary.Read(buf, binary.LittleEndian, &data) + if err != nil { + fmt.Println("binary.Read failed:", err) + } + myFloat = Float{data} + return &myFloat + +} + +func (l Long) String() string { + return fmt.Sprintf("Long Value = %v", l.val) +} + +func (l Long) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, l.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (l Long) Deserialize(values []byte, offset int) *FleetDBType{ + var myLong FleetDBType + var data int64 + buf := bytes.NewReader(values) + err := binary.Read(buf, binary.LittleEndian, &data) + if err != nil { + fmt.Println("binary.Read failed:", err) + } + myLong = Long{data} + return &myLong +} + +func (d Double) String() string{ + return fmt.Sprintf("Double value = %v", d.val) +} + +func (d Double) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, d.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (d Double) Deserialize(values []byte, offset int) *FleetDBType { + var myDouble FleetDBType + var data float64 + buf := bytes.NewReader(values) + err := binary.Read(buf, binary.LittleEndian, &data) + if err != nil { + fmt.Println("binary.Read failed:", err) + } + myDouble = Double{data} + return &myDouble + +} + +func (s String) String() string{ + return fmt.Sprintf("String value = %v", s.val) +} + +func (s String) Serialize() []byte{ + return []byte(s.val) +} + +func (s String) Deserialize(values []byte, offset int) *FleetDBType { + var myString FleetDBType + var data string + data = string(values) + myString = String{data} + return &myString + +} diff --git a/tablestore/fleetDBType_test.go b/tablestore/fleetDBType_test.go index ba4fbee..25ce21e 100644 --- a/tablestore/fleetDBType_test.go +++ b/tablestore/fleetDBType_test.go @@ -12,5 +12,33 @@ func TestSerialize(t *testing.T) { //mySlice := []byte{64,14,3,45} newInt := myInt.Deserialize(byteSlice, 0) fmt.Println((*newInt)) + + myFloat := Float{3.14444} + byteSlice = myFloat.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + newFloat := myFloat.Deserialize(byteSlice, 0) + fmt.Println((*newFloat)) + + myLong := Long{20000000000} + byteSlice = myLong.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + newLong := myLong.Deserialize(byteSlice, 0) + fmt.Println((*newLong)) + + myDouble := Double{3.14444454543546} + byteSlice = myDouble.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + newDouble := myDouble.Deserialize(byteSlice, 0) + fmt.Println((*newDouble)) + + myString := String{"Hello World"} + byteSlice = myString.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + newString := myString.Deserialize(byteSlice, 0) + fmt.Println((*newString)) } From 9fe7af0436d709f51f5ce1ff58558fa65ae7e508 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Sat, 5 Oct 2019 17:57:31 -0400 Subject: [PATCH 07/10] Added Bool Support and prev comment addressed --- tablestore/FleetDBType.go | 150 ------------------------------ tablestore/FleetDBValue.go | 153 +++++++++++++++++++++++++++++++ tablestore/fleetDBType_test.go | 44 --------- tablestore/fleetDBValue_test.go | 57 ++++++++++++ tablestore/table_schema.go | 2 +- tablestore/translatetoKV_test.go | 2 +- 6 files changed, 212 insertions(+), 196 deletions(-) delete mode 100644 tablestore/FleetDBType.go create mode 100644 tablestore/FleetDBValue.go delete mode 100644 tablestore/fleetDBType_test.go create mode 100644 tablestore/fleetDBValue_test.go diff --git a/tablestore/FleetDBType.go b/tablestore/FleetDBType.go deleted file mode 100644 index c703a93..0000000 --- a/tablestore/FleetDBType.go +++ /dev/null @@ -1,150 +0,0 @@ -package tablestore - -import ( - "encoding/binary" - "fmt" - "bytes" -) -type FleetDBType interface{ - Serialize() []byte // this translates to []bytes - Deserialize([]byte, int) *FleetDBType //new type - //FromString(string) *FleetDBType //new from string -} -type Int struct{ - val int32 -} - -type Float struct{ - val float32 -} - -type Long struct{ - val int64 -} - -type Double struct{ - val float64 -} - -type String struct{ - val string -} - -func (i Int) String() string { - return fmt.Sprintf("Integer Value = %v", i.val) -} - -func (i Int) Serialize() []byte{ - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, i.val) - if err != nil { - fmt.Println("binary.Write failed:", err) - } - return buf.Bytes() -} - -func (i Int) Deserialize(values []byte, offset int) *FleetDBType{ - var myInt FleetDBType - var data int32 - buf := bytes.NewReader(values) - err := binary.Read(buf, binary.LittleEndian, &data) - if err != nil { - fmt.Println("binary.Read failed:", err) - } - myInt = Int{data} - return &myInt -} - -func (f Float) String() string{ - return fmt.Sprintf("Float value = %v", f.val) -} - -func (f Float) Serialize() []byte{ - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, f.val) - if err != nil { - fmt.Println("binary.Write failed:", err) - } - return buf.Bytes() -} - -func (f Float) Deserialize(values []byte, offset int) *FleetDBType { - var myFloat FleetDBType - var data float32 - buf := bytes.NewReader(values) - err := binary.Read(buf, binary.LittleEndian, &data) - if err != nil { - fmt.Println("binary.Read failed:", err) - } - myFloat = Float{data} - return &myFloat - -} - -func (l Long) String() string { - return fmt.Sprintf("Long Value = %v", l.val) -} - -func (l Long) Serialize() []byte{ - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, l.val) - if err != nil { - fmt.Println("binary.Write failed:", err) - } - return buf.Bytes() -} - -func (l Long) Deserialize(values []byte, offset int) *FleetDBType{ - var myLong FleetDBType - var data int64 - buf := bytes.NewReader(values) - err := binary.Read(buf, binary.LittleEndian, &data) - if err != nil { - fmt.Println("binary.Read failed:", err) - } - myLong = Long{data} - return &myLong -} - -func (d Double) String() string{ - return fmt.Sprintf("Double value = %v", d.val) -} - -func (d Double) Serialize() []byte{ - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, d.val) - if err != nil { - fmt.Println("binary.Write failed:", err) - } - return buf.Bytes() -} - -func (d Double) Deserialize(values []byte, offset int) *FleetDBType { - var myDouble FleetDBType - var data float64 - buf := bytes.NewReader(values) - err := binary.Read(buf, binary.LittleEndian, &data) - if err != nil { - fmt.Println("binary.Read failed:", err) - } - myDouble = Double{data} - return &myDouble - -} - -func (s String) String() string{ - return fmt.Sprintf("String value = %v", s.val) -} - -func (s String) Serialize() []byte{ - return []byte(s.val) -} - -func (s String) Deserialize(values []byte, offset int) *FleetDBType { - var myString FleetDBType - var data string - data = string(values) - myString = String{data} - return &myString - -} diff --git a/tablestore/FleetDBValue.go b/tablestore/FleetDBValue.go new file mode 100644 index 0000000..c275145 --- /dev/null +++ b/tablestore/FleetDBValue.go @@ -0,0 +1,153 @@ +package tablestore + +import ( + "io" + "encoding/binary" + "fmt" + "bytes" +) +type FleetDBValue interface{ + Serialize() []byte // this translates to []bytes + Deserialize(io.Reader) //new type + //FromString(string) *FleetDBValue //new from string +} +type IntValue struct{ + val int32 +} + +type FloatValue struct{ + val float32 +} + +type BigIntValue struct{ + val int64 +} + +type DoubleValue struct{ + val float64 +} + +type TextValue struct{ + val string +} + +type BooleanValue struct{ + val bool +} + +func (i IntValue) String() string { + return fmt.Sprintf("Integer Value = %v", i.val) +} + +func (i IntValue) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, i.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (i IntValue) Deserialize(reader io.Reader){ + err := binary.Read(reader, binary.LittleEndian, &(i.val)) + if err != nil { + fmt.Println("binary.Read failed:", err) + } +} + +func (f FloatValue) String() string{ + return fmt.Sprintf("FloatValue = %v", f.val) +} + +func (f FloatValue) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, f.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (f FloatValue) Deserialize(reader io.Reader){ + err := binary.Read(reader, binary.LittleEndian, &(f.val)) + if err != nil { + fmt.Println("binary.Read failed:", err) + } +} + +func (l BigIntValue) String() string { + return fmt.Sprintf("BigIntValue Value = %v", l.val) +} + +func (l BigIntValue) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, l.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (l BigIntValue) Deserialize(reader io.Reader){ + err := binary.Read(reader, binary.LittleEndian, &(l.val)) + if err != nil { + fmt.Println("binary.Read failed:", err) + } +} + +func (d DoubleValue) String() string{ + return fmt.Sprintf("DoubleValue = %v", d.val) +} + +func (d DoubleValue) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, d.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (d DoubleValue) Deserialize(reader io.Reader){ + err := binary.Read(reader, binary.LittleEndian, &(d.val)) + if err != nil { + fmt.Println("binary.Read failed:", err) + } +} + +func (s TextValue) String() string{ + return fmt.Sprintf("String value = %v", s.val) +} + +func (s TextValue) Serialize() []byte{ + return []byte(s.val) +} + +func (s TextValue) Deserialize(reader io.Reader){ + var value []byte + err := binary.Read(reader, binary.LittleEndian, value) + if err != nil { + fmt.Println("binary.Read failed:", err) + } + s.val = string(value) +} + +func (b BooleanValue) String() string{ + return fmt.Sprintf("BooleanValue = %v", b.val) +} + +func (b BooleanValue) Serialize() []byte{ + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, b.val) + if err != nil { + fmt.Println("binary.Write failed:", err) + } + return buf.Bytes() +} + +func (b BooleanValue) Deserialize(reader io.Reader){ + err := binary.Read(reader, binary.LittleEndian, &(b.val)) + if err != nil { + fmt.Println("binary.Read failed:", err) + } +} \ No newline at end of file diff --git a/tablestore/fleetDBType_test.go b/tablestore/fleetDBType_test.go deleted file mode 100644 index 25ce21e..0000000 --- a/tablestore/fleetDBType_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package tablestore - -import ( - "testing" - "fmt" -) - -func TestSerialize(t *testing.T) { - myInt := Int{200000} - byteSlice := myInt.Serialize() - fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} - newInt := myInt.Deserialize(byteSlice, 0) - fmt.Println((*newInt)) - - myFloat := Float{3.14444} - byteSlice = myFloat.Serialize() - fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} - newFloat := myFloat.Deserialize(byteSlice, 0) - fmt.Println((*newFloat)) - - myLong := Long{20000000000} - byteSlice = myLong.Serialize() - fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} - newLong := myLong.Deserialize(byteSlice, 0) - fmt.Println((*newLong)) - - myDouble := Double{3.14444454543546} - byteSlice = myDouble.Serialize() - fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} - newDouble := myDouble.Deserialize(byteSlice, 0) - fmt.Println((*newDouble)) - - myString := String{"Hello World"} - byteSlice = myString.Serialize() - fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} - newString := myString.Deserialize(byteSlice, 0) - fmt.Println((*newString)) -} - diff --git a/tablestore/fleetDBValue_test.go b/tablestore/fleetDBValue_test.go new file mode 100644 index 0000000..69e6677 --- /dev/null +++ b/tablestore/fleetDBValue_test.go @@ -0,0 +1,57 @@ +package tablestore + +import ( + "testing" + "fmt" + "bytes" +) + +func TestSerialize(t *testing.T) { + myInt := IntValue{200000} + byteSlice := myInt.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + reader := bytes.NewReader(byteSlice) + myInt.Deserialize(reader) + fmt.Println(myInt.val) + + myFloat := FloatValue{3.14444} + byteSlice = myFloat.Serialize() + fmt.Println(byteSlice) + reader = bytes.NewReader(byteSlice) + myFloat.Deserialize(reader) + fmt.Println(myFloat.val) + + myLong := BigIntValue{20000000000} + byteSlice = myLong.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + reader = bytes.NewReader(byteSlice) + myLong.Deserialize(reader) + fmt.Println(myLong.val) + + myDouble := DoubleValue{3.14444454543546} + byteSlice = myDouble.Serialize() + fmt.Println(byteSlice) + //mySlice := []byte{64,14,3,45} + reader = bytes.NewReader(byteSlice) + myDouble.Deserialize(reader) + fmt.Println(myDouble.val) + + myString := TextValue{"Hello World"} + byteSlice = myString.Serialize() + fmt.Println(byteSlice) + reader = bytes.NewReader(byteSlice) + //mySlice := []byte{64,14,3,45} + myString.Deserialize(reader) + fmt.Println(myString.val) + + myBoolean := BooleanValue{true} + byteSlice = myBoolean.Serialize() + fmt.Println(byteSlice) + reader = bytes.NewReader(byteSlice) + //mySlice := []byte{64,14,3,45} + myBoolean.Deserialize(reader) + fmt.Println(myBoolean.val) +} + diff --git a/tablestore/table_schema.go b/tablestore/table_schema.go index cc3cc1b..4e2a33b 100644 --- a/tablestore/table_schema.go +++ b/tablestore/table_schema.go @@ -6,7 +6,7 @@ import ( type FleetDbColumnSpec struct { colname string - coltype FleetDBType + coltype FleetDBValue isPartition bool isClustering bool } diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index ff499de..a843803 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -15,7 +15,7 @@ func TestTranslateToKV(t *testing.T) { tableMap := make(map[string][]FleetDbColumnSpec) tableName := "crossfit_gyms" - var myInt Int + var myInt IntValue //Test With One Partition Key and One Clustering Key myschema := make([]FleetDbColumnSpec, 4) myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} From 2ea9ab2e938bc311ba0d1b05daba394ca8ee4956 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Tue, 8 Oct 2019 13:04:04 -0400 Subject: [PATCH 08/10] Refactoring --- tablestore/fleetDBValue_test.go | 29 ++++++++++++++++++----------- tablestore/translatetoKV_test.go | 19 ++++--------------- tablestore/utility.go | 16 ++++------------ 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/tablestore/fleetDBValue_test.go b/tablestore/fleetDBValue_test.go index 69e6677..ac1ba8e 100644 --- a/tablestore/fleetDBValue_test.go +++ b/tablestore/fleetDBValue_test.go @@ -10,48 +10,55 @@ func TestSerialize(t *testing.T) { myInt := IntValue{200000} byteSlice := myInt.Serialize() fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} reader := bytes.NewReader(byteSlice) myInt.Deserialize(reader) - fmt.Println(myInt.val) + if myInt.val != 200000 { + t.Errorf("Int Serialization DeSerialization failed, expected %v, got %v", "200000" , myInt.val) + } myFloat := FloatValue{3.14444} byteSlice = myFloat.Serialize() fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) myFloat.Deserialize(reader) - fmt.Println(myFloat.val) + if myFloat.val != 3.14444 { + t.Errorf("Float Serialization DeSerialization failed, expected %v, got %v", "3.14444" , myFloat.val) + } myLong := BigIntValue{20000000000} byteSlice = myLong.Serialize() fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} reader = bytes.NewReader(byteSlice) myLong.Deserialize(reader) - fmt.Println(myLong.val) + if myLong.val != 20000000000 { + t.Errorf("Long Serialization DeSerialization failed, expected %v, got %v", "20000000000" , myLong.val) + } myDouble := DoubleValue{3.14444454543546} byteSlice = myDouble.Serialize() fmt.Println(byteSlice) - //mySlice := []byte{64,14,3,45} reader = bytes.NewReader(byteSlice) myDouble.Deserialize(reader) - fmt.Println(myDouble.val) + if myDouble.val != 3.14444454543546 { + t.Errorf("Double Serialization DeSerialization failed, expected %v, got %v", "3.14444454543546" , myDouble.val) + } myString := TextValue{"Hello World"} byteSlice = myString.Serialize() fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - //mySlice := []byte{64,14,3,45} myString.Deserialize(reader) - fmt.Println(myString.val) + if myString.val != "Hello World" { + t.Errorf("String Serialization DeSerialization failed, expected %v, got %v", "Hello World" , myString.val) + } myBoolean := BooleanValue{true} byteSlice = myBoolean.Serialize() fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - //mySlice := []byte{64,14,3,45} myBoolean.Deserialize(reader) - fmt.Println(myBoolean.val) + if myBoolean.val != true { + t.Errorf("Boolean Serialization DeSerialization failed, expected %v, got %v", "true" , myBoolean.val) + } } diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index a843803..80a78df 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -6,23 +6,19 @@ import ( ) func TestTranslateToKV(t *testing.T) { - - - //createCommand := "CREATE TABLE crossfit_gyms (country_code string,state_province string,city string,gym_name string,PRIMARY KEY (country_code, state_province));" - //fmt.Println(createCommand) - //myschema,tableName := createSchemaOnePOneC(createCommand) tableMap := make(map[string][]FleetDbColumnSpec) tableName := "crossfit_gyms" var myInt IntValue + + //Test With One Partition Key and One Clustering Key myschema := make([]FleetDbColumnSpec, 4) myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, true} myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} - tableMap[tableName] = myschema insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; rowData, myTableName := decodeInsertCommand(insertCommand) @@ -39,7 +35,6 @@ func TestTranslateToKV(t *testing.T) { if string(res[1].Value) != "University Avenue" { t.Errorf("TranslateToKV() failed, expected %v, got %v", "University Avenue" , string(res[1].Value)) } - //fmt.Println(string(res[0].Key)); //Test With One Partition Key and Zero Clustering Key @@ -48,7 +43,6 @@ func TestTranslateToKV(t *testing.T) { myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, false} myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} - tableMap[tableName] = myschema insertCommand = "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; rowData, myTableName = decodeInsertCommand(insertCommand) @@ -71,15 +65,10 @@ func TestTranslateToKV(t *testing.T) { if string(res[2].Value) != "University Avenue" { t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "University Avenue" , string(res[2].Value)) } - - - - // - } -func decodeInsertCommand(query string)([][]byte , string){ - val := [][]byte{[]byte("US"), []byte("NY"),[]byte("Buffalo"),[]byte("University Avenue")} +func decodeInsertCommand(query string)([]FleetDBValue , string){ + val := []FleetDBValue{TextValue{"US"}, TextValue{"NY"},TextValue{"Buffalo"},TextValue{"University Avenue"}} return val, "crossfit_gyms" } diff --git a/tablestore/utility.go b/tablestore/utility.go index 38bdac2..a03cc8a 100644 --- a/tablestore/utility.go +++ b/tablestore/utility.go @@ -1,24 +1,16 @@ package tablestore import ( - //"fmt" - //"strings" - //"github.com/darshannevgi/fleetdb/kv_store" - //"github.com/darshannevgi/fleetdb/tablestore" ) -/* -Each values[0] represents byte[] data value of first column -if first column is country_code then values[0] is byte representation of 'US' = byte[]{85,83} -*/ -func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ +func TranslateToKV(columnSpecs []FleetDbColumnSpec, values []FleetDBValue) []KVItem{ var pKey []byte var cKey []byte for i, colSpec := range columnSpecs { if colSpec.isPartition{ - pKey = append(pKey, values[i]...) + pKey = append(pKey, values[i].Serialize()...) } if colSpec.isClustering{ - cKey = append(cKey, values[i]...) + cKey = append(cKey, values[i].Serialize()...) } } //Handle composite key and only pK and cK case @@ -38,7 +30,7 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values [][]byte) []KVItem{ for i, colSpec := range columnSpecs { if !colSpec.isPartition && !colSpec.isClustering{ key := append(prefix,colSpec.colname...) - val := values[i] + val := values[i].Serialize() //fmt.Println("Key is =" + string(key)) //fmt.Println("Value is =" + string(val)) kvItems[index] = KVItem{key, val} From 2f592bc76b24923b84b3f421e20fb62d9209ba63 Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Tue, 8 Oct 2019 21:50:30 -0400 Subject: [PATCH 09/10] Assert support and Extra Corner test cases added --- tablestore/FleetDBValue.go | 93 +++++++++++++++++++++++---- tablestore/fleetDBValue_test.go | 52 ++++++--------- tablestore/translatetoKV_test.go | 99 ++++++++++++++++------------- tablestore/utility.go | 106 +++++++++++++++++++++++++------ 4 files changed, 238 insertions(+), 112 deletions(-) diff --git a/tablestore/FleetDBValue.go b/tablestore/FleetDBValue.go index c275145..999e189 100644 --- a/tablestore/FleetDBValue.go +++ b/tablestore/FleetDBValue.go @@ -5,12 +5,30 @@ import ( "encoding/binary" "fmt" "bytes" + "io/ioutil" ) type FleetDBValue interface{ Serialize() []byte // this translates to []bytes - Deserialize(io.Reader) //new type + //Deserialize(io.Reader) //new type + String() string + getType() FleetDBType //FromString(string) *FleetDBValue //new from string } + +type FleetDBType uint8 +const ( + Int FleetDBType = iota + 1 + BigInt + Float + Double + Text + Boolean +) + +func (f FleetDBType) getVal() uint8{ + return uint8(f) +} + type IntValue struct{ val int32 } @@ -48,11 +66,17 @@ func (i IntValue) Serialize() []byte{ return buf.Bytes() } -func (i IntValue) Deserialize(reader io.Reader){ +func NewIntValue(reader io.Reader) *IntValue{ + var i IntValue err := binary.Read(reader, binary.LittleEndian, &(i.val)) if err != nil { fmt.Println("binary.Read failed:", err) } + return &i +} + +func (i IntValue) getType() FleetDBType{ + return Int; } func (f FloatValue) String() string{ @@ -68,11 +92,17 @@ func (f FloatValue) Serialize() []byte{ return buf.Bytes() } -func (f FloatValue) Deserialize(reader io.Reader){ +func NewFloatValue(reader io.Reader) *FloatValue{ + var f FloatValue err := binary.Read(reader, binary.LittleEndian, &(f.val)) if err != nil { fmt.Println("binary.Read failed:", err) - } + } + return &f +} + +func (f FloatValue) getType() FleetDBType{ + return Float } func (l BigIntValue) String() string { @@ -88,11 +118,17 @@ func (l BigIntValue) Serialize() []byte{ return buf.Bytes() } -func (l BigIntValue) Deserialize(reader io.Reader){ +func NewBigIntValue(reader io.Reader) *BigIntValue{ + var l BigIntValue err := binary.Read(reader, binary.LittleEndian, &(l.val)) if err != nil { fmt.Println("binary.Read failed:", err) } + return &l +} + +func (l BigIntValue) getType() FleetDBType{ + return BigInt } func (d DoubleValue) String() string{ @@ -108,11 +144,17 @@ func (d DoubleValue) Serialize() []byte{ return buf.Bytes() } -func (d DoubleValue) Deserialize(reader io.Reader){ +func NewDoubleValue(reader io.Reader) *DoubleValue{ + var d DoubleValue err := binary.Read(reader, binary.LittleEndian, &(d.val)) if err != nil { fmt.Println("binary.Read failed:", err) - } + } + return &d +} + +func (d DoubleValue) getType() FleetDBType{ + return Double } func (s TextValue) String() string{ @@ -123,13 +165,30 @@ func (s TextValue) Serialize() []byte{ return []byte(s.val) } -func (s TextValue) Deserialize(reader io.Reader){ - var value []byte - err := binary.Read(reader, binary.LittleEndian, value) +func NewTextValue(reader io.Reader) *TextValue{ + b , err := ioutil.ReadAll(reader) if err != nil { - fmt.Println("binary.Read failed:", err) + fmt.Println("readAll failed:", err) + } + //fmt.Println(string(b)) + ans := TextValue{string(b)} + return &ans +} + +func NewTextValueFromNullTerminatedStream(r io.Reader) *TextValue { + p := []byte{} + one := make([]byte, 1) + _ , err := r.Read(one) + for err != io.EOF && string(one) != "\000" { + p = append(p,one...) + _ , err = r.Read(one) } - s.val = string(value) + ans := TextValue{string(p)} + return &ans +} + +func (s TextValue) getType() FleetDBType{ + return Text } func (b BooleanValue) String() string{ @@ -145,9 +204,15 @@ func (b BooleanValue) Serialize() []byte{ return buf.Bytes() } -func (b BooleanValue) Deserialize(reader io.Reader){ +func NewBooleanValue(reader io.Reader) *BooleanValue{ + var b BooleanValue err := binary.Read(reader, binary.LittleEndian, &(b.val)) if err != nil { fmt.Println("binary.Read failed:", err) - } + } + return &b +} + +func (b BooleanValue) getType() FleetDBType{ + return Boolean } \ No newline at end of file diff --git a/tablestore/fleetDBValue_test.go b/tablestore/fleetDBValue_test.go index ac1ba8e..fe5087c 100644 --- a/tablestore/fleetDBValue_test.go +++ b/tablestore/fleetDBValue_test.go @@ -2,63 +2,51 @@ package tablestore import ( "testing" - "fmt" + // "fmt" "bytes" + "github.com/stretchr/testify/assert" ) func TestSerialize(t *testing.T) { myInt := IntValue{200000} byteSlice := myInt.Serialize() - fmt.Println(byteSlice) + //fmt.Println(byteSlice) reader := bytes.NewReader(byteSlice) - myInt.Deserialize(reader) - if myInt.val != 200000 { - t.Errorf("Int Serialization DeSerialization failed, expected %v, got %v", "200000" , myInt.val) - } + assert.Equal(t, int32(200000), NewIntValue(reader).val) myFloat := FloatValue{3.14444} byteSlice = myFloat.Serialize() - fmt.Println(byteSlice) + //fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - myFloat.Deserialize(reader) - if myFloat.val != 3.14444 { - t.Errorf("Float Serialization DeSerialization failed, expected %v, got %v", "3.14444" , myFloat.val) - } - + assert.Equal(t, float32(3.14444), NewFloatValue(reader).val) + myLong := BigIntValue{20000000000} byteSlice = myLong.Serialize() - fmt.Println(byteSlice) + //fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - myLong.Deserialize(reader) - if myLong.val != 20000000000 { - t.Errorf("Long Serialization DeSerialization failed, expected %v, got %v", "20000000000" , myLong.val) - } + assert.Equal(t, int64(20000000000), NewBigIntValue(reader).val) myDouble := DoubleValue{3.14444454543546} byteSlice = myDouble.Serialize() - fmt.Println(byteSlice) + //fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - myDouble.Deserialize(reader) - if myDouble.val != 3.14444454543546 { - t.Errorf("Double Serialization DeSerialization failed, expected %v, got %v", "3.14444454543546" , myDouble.val) - } + assert.Equal(t, float64(3.14444454543546), NewDoubleValue(reader).val) myString := TextValue{"Hello World"} byteSlice = myString.Serialize() - fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - myString.Deserialize(reader) - if myString.val != "Hello World" { - t.Errorf("String Serialization DeSerialization failed, expected %v, got %v", "Hello World" , myString.val) - } + assert.Equal(t, "Hello World", NewTextValue(reader).val) + + myNullTerminatedString := TextValue{"Hello World\000"} + byteSlice = myNullTerminatedString.Serialize() + reader = bytes.NewReader(byteSlice) + //fmt.Println(NewTextValueFromNullTerminatedStream(reader).val) + assert.Equal(t, "Hello World", NewTextValueFromNullTerminatedStream(reader).val) myBoolean := BooleanValue{true} byteSlice = myBoolean.Serialize() - fmt.Println(byteSlice) + //fmt.Println(byteSlice) reader = bytes.NewReader(byteSlice) - myBoolean.Deserialize(reader) - if myBoolean.val != true { - t.Errorf("Boolean Serialization DeSerialization failed, expected %v, got %v", "true" , myBoolean.val) - } + assert.Equal(t, true, NewBooleanValue(reader).val) } diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index 80a78df..55e4a78 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -2,7 +2,8 @@ package tablestore import ( "testing" - // "fmt" + // "github.com/stretchr/testify/assert" + //"fmt" ) func TestTranslateToKV(t *testing.T) { @@ -10,61 +11,68 @@ func TestTranslateToKV(t *testing.T) { tableMap := make(map[string][]FleetDbColumnSpec) tableName := "crossfit_gyms" - var myInt IntValue + var myText TextValue //Test With One Partition Key and One Clustering Key myschema := make([]FleetDbColumnSpec, 4) - myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, true} - myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} tableMap[tableName] = myschema insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; rowData, myTableName := decodeInsertCommand(insertCommand) res := TranslateToKV(tableMap[myTableName], rowData) - if string(res[0].Key) != "US/NY/city" { - t.Errorf("TranslateToKV() failed, expected %v, got %v", "US/NY/city" , string(res[0].Key)) - } - if string(res[0].Value) != "Buffalo" { - t.Errorf("TranslateToKV() failed, expected %v, got %v", "Buffalo" , string(res[0].Value)) - } - if string(res[1].Key) != "US/NY/gym_name" { - t.Errorf("TranslateToKV() failed, expected %v, got %v", "US/NY/gym_name" , string(res[1].Key)) - } - if string(res[1].Value) != "University Avenue" { - t.Errorf("TranslateToKV() failed, expected %v, got %v", "University Avenue" , string(res[1].Value)) - } - + parseKey(res) + //fmt.Println(res[0].Key) + //fmt.Println(res[1].Key) + //fmt.Println(string([]byte{3,5,99,105,116,121,0})) + //assert.Equal(t, "US/NY/city", string(res[0].Key)) + //assert.Equal(t, "Buffalo", string(res[0].Value)) + //assert.Equal(t, "US/NY/gym_name", string(res[1].Key)) + //assert.Equal(t, "University Avenue", string(res[1].Value)) + /* //Test With One Partition Key and Zero Clustering Key - myschema = make([]FleetDbColumnSpec, 4) - myschema[0] = FleetDbColumnSpec{"country_code", myInt, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myInt, false, false} - myschema[2] = FleetDbColumnSpec{"city", myInt, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", myInt, false, false} - tableMap[tableName] = myschema - insertCommand = "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; - rowData, myTableName = decodeInsertCommand(insertCommand) + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, false} res = TranslateToKV(tableMap[myTableName], rowData) - if string(res[0].Key) != "US/state_province" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/state_province" , string(res[0].Key)) - } - if string(res[0].Value) != "NY" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "NY" , string(res[0].Value)) - } - if string(res[1].Key) != "US/city" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/city" , string(res[1].Key)) - } - if string(res[1].Value) != "Buffalo" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "Buffalo" , string(res[1].Value)) - } - if string(res[2].Key) != "US/gym_name" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "US/gym_name" , string(res[2].Key)) - } - if string(res[2].Value) != "University Avenue" { - t.Errorf("TranslateToKV() Test case 2 failed, expected %v, got %v", "University Avenue" , string(res[2].Value)) - } + assert.Equal(t, "US/state_province", string(res[0].Key)) + assert.Equal(t, "NY", string(res[0].Value)) + assert.Equal(t, "US/city", string(res[1].Key)) + assert.Equal(t, "Buffalo", string(res[1].Value)) + assert.Equal(t, "US/gym_name", string(res[2].Key)) + assert.Equal(t, "University Avenue", string(res[2].Value)) + + //All columns are either PK or CK + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} + myschema[2] = FleetDbColumnSpec{"city", myText, false, true} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, true} + res = TranslateToKV(tableMap[myTableName], rowData) + assert.Equal(t, "US/NY/Buffalo/University Avenue", string(res[0].Key)) + assert.Equal(t,[]byte(nil), res[0].Value) + + // Composite Key + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, true, false} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} + res = TranslateToKV(tableMap[myTableName], rowData) + assert.Equal(t, "US/NY/city", string(res[0].Key)) + assert.Equal(t, "Buffalo", string(res[0].Value)) + assert.Equal(t, "US/NY/gym_name", string(res[1].Key)) + assert.Equal(t, "University Avenue", string(res[1].Value)) + + //Composite Key with Clustering Key + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} + myschema[2] = FleetDbColumnSpec{"city", myText, true, false} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} + res = TranslateToKV(tableMap[myTableName], rowData) + assert.Equal(t, "US/Buffalo/NY/gym_name", string(res[0].Key)) + assert.Equal(t, "University Avenue", string(res[0].Value)) + */ } func decodeInsertCommand(query string)([]FleetDBValue , string){ @@ -72,6 +80,7 @@ func decodeInsertCommand(query string)([]FleetDBValue , string){ return val, "crossfit_gyms" } + /*func createSchema(query string) ([]FleetDbColumnSpec,string) { schema := make([]FleetDbColumnSpec, 4) schema[0] = FleetDbColumnSpec{"country_code", string, true, false} diff --git a/tablestore/utility.go b/tablestore/utility.go index a03cc8a..4ccb583 100644 --- a/tablestore/utility.go +++ b/tablestore/utility.go @@ -1,41 +1,105 @@ package tablestore import ( + "fmt" + "io" + "bytes" + //"encoding/binary" ) + +const( + PartitionType = iota + 1; + ClusteringType + ColumnType +) + +var nullchar string = "\000" + func TranslateToKV(columnSpecs []FleetDbColumnSpec, values []FleetDBValue) []KVItem{ var pKey []byte var cKey []byte + pKeyCount := 0 + cKeyCount := 0 for i, colSpec := range columnSpecs { if colSpec.isPartition{ - pKey = append(pKey, values[i].Serialize()...) + pKey = append(pKey,PartitionType) + coltype := colSpec.coltype.getType() + pKey = append(pKey, coltype.getVal()) + if coltype == Text{ + pKey = append(pKey,[]byte(values[i].String()+ nullchar)...) + }else{ + pKey = append(pKey,values[i].Serialize()...) + } + pKeyCount = pKeyCount + 1 } + if colSpec.isClustering{ - cKey = append(cKey, values[i].Serialize()...) + cKey = append(cKey,ClusteringType) + coltype := colSpec.coltype.getType() + cKey = append(cKey, coltype.getVal()) + if coltype == Text{ + cKey = append(cKey,[]byte(values[i].String() + nullchar)...) + }else{ + cKey = append(cKey,values[i].Serialize()...) + } + cKeyCount = cKeyCount + 1 } } - //Handle composite key and only pK and cK case - //In case of composite key , Key is made in order of partition key specified in create table command - //if PRIMARY KEY ((country_code, state_province, city) then key will be US/NY/Buffalo + //Handle Nil values - kvItems := make([]KVItem, len(values)) - index := 0 + kvItems := []KVItem{} var prefix []byte + //var key []byte prefix = pKey - prefix = append(prefix,"/"...) - if len(cKey) > 0{ + + if cKeyCount > 0{ prefix = append(prefix,cKey...) - prefix = append(prefix,"/"...) } - - for i, colSpec := range columnSpecs { - if !colSpec.isPartition && !colSpec.isClustering{ - key := append(prefix,colSpec.colname...) - val := values[i].Serialize() - //fmt.Println("Key is =" + string(key)) - //fmt.Println("Value is =" + string(val)) - kvItems[index] = KVItem{key, val} - index = index + 1 - } - } + if len(columnSpecs) == (pKeyCount + cKeyCount){ + key := prefix + var val []byte + kvItems[0] = KVItem{key, val} + }else{ + //index := 0 + for i, colSpec := range columnSpecs { + if !colSpec.isPartition && !colSpec.isClustering{ + //fmt.Println(index) + key := []byte{} + key = append(key,prefix...) + key = append(key,ColumnType) + key = append(key, Text.getVal()) + key = append(key,[]byte(colSpec.colname + nullchar)...) + val := values[i].Serialize() + kvItems = append(kvItems, KVItem{key, val}) + } + } + } return kvItems; } + +//Incomplete Implementation +func parseKey(data []KVItem){ + for _, item := range data{ + r := bytes.NewReader(item.Key) + p := make([]byte, 1) + //var i uint8 + //fmt.Println(string(item.Key)) + _ , err := r.Read(p) + for err != io.EOF{ + r.Read(p) + //r.Read(p) + switch p[0] { + case 1: + fmt.Printf("Integer") + case 2: + fmt.Printf("BigInt") + case 5: + //fmt.Printf("Text \n") + text := NewTextValueFromNullTerminatedStream(r) + fmt.Println(text.val) + } + _ , err = r.Read(p) + } + } +} + From 996f39e3d0620a37b0df8610c973d4ce5898d61b Mon Sep 17 00:00:00 2001 From: Darshan Nevgi Date: Wed, 23 Oct 2019 19:59:05 -0400 Subject: [PATCH 10/10] Added Basic Functionalities of Save and GetPartition --- tablestore/FleetDBTableSpec.go | 16 + tablestore/FleetDBValue.go | 22 +- tablestore/database.go | 234 ++++++++++ tablestore/db_test.go | 415 ++++++++++++++++++ .../000001.log | Bin 0 -> 49 bytes .../CURRENT | 1 + .../25431cf8-3e99-460e-be81-f784a1400873/LOCK | 0 .../25431cf8-3e99-460e-be81-f784a1400873/LOG | 6 + .../MANIFEST-000000 | Bin 0 -> 54 bytes .../000001.log | Bin 0 -> 49 bytes .../CURRENT | 1 + .../ad50d105-cdec-44f5-9eba-e2537944cd26/LOCK | 0 .../ad50d105-cdec-44f5-9eba-e2537944cd26/LOG | 6 + .../MANIFEST-000000 | Bin 0 -> 54 bytes tablestore/row.go | 9 + tablestore/table_schema.go | 1 + tablestore/translatetoKV_test.go | 140 +++--- tablestore/utility.go | 44 +- 18 files changed, 796 insertions(+), 99 deletions(-) create mode 100644 tablestore/FleetDBTableSpec.go create mode 100644 tablestore/database.go create mode 100644 tablestore/db_test.go create mode 100644 tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/000001.log create mode 100644 tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/CURRENT create mode 100644 tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOCK create mode 100644 tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOG create mode 100644 tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/MANIFEST-000000 create mode 100644 tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/000001.log create mode 100644 tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/CURRENT create mode 100644 tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOCK create mode 100644 tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOG create mode 100644 tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/MANIFEST-000000 create mode 100644 tablestore/row.go diff --git a/tablestore/FleetDBTableSpec.go b/tablestore/FleetDBTableSpec.go new file mode 100644 index 0000000..6497ef8 --- /dev/null +++ b/tablestore/FleetDBTableSpec.go @@ -0,0 +1,16 @@ +package tablestore + +import ( + "fmt" +) +type FleetDBTableSpec struct { + columnSpecs map[string]*FleetDbColumnSpec +} + +func (s *FleetDBTableSpec) GetColumnSpec(column string) *FleetDbColumnSpec { + if column == ""{ + fmt.Println("Table name is Nil") + return nil + } + return s.columnSpecs[column] +} diff --git a/tablestore/FleetDBValue.go b/tablestore/FleetDBValue.go index 999e189..00bc596 100644 --- a/tablestore/FleetDBValue.go +++ b/tablestore/FleetDBValue.go @@ -9,10 +9,8 @@ import ( ) type FleetDBValue interface{ Serialize() []byte // this translates to []bytes - //Deserialize(io.Reader) //new type String() string getType() FleetDBType - //FromString(string) *FleetDBValue //new from string } type FleetDBType uint8 @@ -25,6 +23,23 @@ const ( Boolean ) +type ComparatorType uint8 +const ( + LESSTHAN = iota + 1 + LESSTHANEQUAL + GREATERTHAN + GREATERTHANEQUAL + BETWEEN + EQUAL +) + +type ColKeyType uint8 +const ( + PRIMARY ColKeyType = iota + 1 + CLUSTERING + COLUMN +) + func (f FleetDBType) getVal() uint8{ return uint8(f) } @@ -158,7 +173,7 @@ func (d DoubleValue) getType() FleetDBType{ } func (s TextValue) String() string{ - return fmt.Sprintf("String value = %v", s.val) + return fmt.Sprintf(s.val) } func (s TextValue) Serialize() []byte{ @@ -170,7 +185,6 @@ func NewTextValue(reader io.Reader) *TextValue{ if err != nil { fmt.Println("readAll failed:", err) } - //fmt.Println(string(b)) ans := TextValue{string(b)} return &ans } diff --git a/tablestore/database.go b/tablestore/database.go new file mode 100644 index 0000000..3e464ae --- /dev/null +++ b/tablestore/database.go @@ -0,0 +1,234 @@ +package tablestore + +import ( + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" + "github.com/google/uuid" + "sync" + "fmt" + "io" + "bytes" +) + +// StateMachine interface provides execution of command against database +// the implementation should be thread safe +type Store interface { + Save(tableid uuid.UUID, items []KVItem) error + GetPartition(tableid uuid.UUID, partitionKeys []FleetDBValue, clusteringKyes []FleetDBValue, clusteringComparators []ComparatorType, columns []string, tableSpec FleetDBTableSpec) ([]Row, error) +} + +// database maintains the key-value datastore +type database struct { + lock *sync.RWMutex + leveldbs map[uuid.UUID] *leveldb.DB // map of leveldb shards +} + +// NewStore get the instance of LevelDB Wrapper +func NewStore() Store { + db := new(database) + db.lock = new(sync.RWMutex) + db.leveldbs = make(map[uuid.UUID] *leveldb.DB) + return db +} + +func (db *database) getStore(tableid uuid.UUID) *leveldb.DB{ + lvldb := db.leveldbs[tableid] + dir := "/tmp/lvldb" + if lvldb == nil { + lvlDBName := dir + "/" + tableid.String() + lvldb, _ = leveldb.OpenFile(lvlDBName,nil) + db.leveldbs[tableid] = lvldb; + } + return lvldb + +} + +func (db *database) Save(tableid uuid.UUID, items []KVItem) error { + storedb := db.getStore(tableid) + var err error + batch := new(leveldb.Batch) + for _, item := range items{ + if(item.Value == nil){ + fmt.Printf("Deleting Key %s", item.Key) + batch.Delete(item.Key) + }else{ + batch.Put(item.Key,item.Value) + } + } + err = storedb.Write(batch, nil) + if err != nil { + return err; + } + return nil +} + + +func buildKey(keyType uint8, key FleetDBValue) []byte{ + searchKey := make([]byte,0) + searchKey = append(searchKey, keyType) + coltype := key.getType() + searchKey = append(searchKey, coltype.getVal()) + if coltype == Text{ + searchKey = append(searchKey,[]byte(key.String()+ nullchar)...) + }else{ + searchKey = append(searchKey,key.Serialize()...) + } + return searchKey +} + +func (db *database) GetPartition(tableid uuid.UUID, partitionKeys []FleetDBValue , clusteringKyes []FleetDBValue, clusteringComparators []ComparatorType, columns []string, tableSpec FleetDBTableSpec) ([]Row, error) { + var rows []Row + var err error + storedb := db.getStore(tableid) + oldCKeyvalues := []FleetDBValue{} + oldColvalues := []FleetDBValue{} + seqMap := make(map[uint8]string) + for colName , columnSpec := range tableSpec.columnSpecs{ + seqMap[columnSpec.seqNo] = colName + } + + valMap := make(map[string]FleetDBValue) + oldValMap := make(map[string]FleetDBValue) + searchKey := make([]byte,0) + for _ , key := range partitionKeys{ + searchKey = append(searchKey,buildKey(PartitionType, key)...) + } + index := 0 + for index < len(clusteringComparators) && clusteringComparators[index] == EQUAL{ + searchKey = append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + index++ + } + isAtFirst := false + iter := storedb.NewIterator(util.BytesPrefix(searchKey), nil) + if(index < len(clusteringComparators)){ + switch(clusteringComparators[index]){ + case GREATERTHANEQUAL: + lowerLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + iter = storedb.NewIterator(nil, nil) + iter.Seek(lowerLimit) + isAtFirst = true + case GREATERTHAN: + lowerLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + iter = storedb.NewIterator(nil, nil) + iter.Seek(lowerLimit) + isAtFirst = true + for (isAtFirst || iter.Next()) && bytes.HasPrefix(iter.Key(), lowerLimit){ + isAtFirst = false + } + isAtFirst = true + case LESSTHAN: + lowerLimit := append(searchKey, ClusteringType) + lowerLimit = append(lowerLimit, clusteringKyes[index].getType().getVal()) + upperLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + iter = storedb.NewIterator(&util.Range{Start: lowerLimit, Limit: upperLimit}, nil) + //fmt.Printf("LessThanEqualLower: %s LessThanEqualUpper: %s ", NewTextValue(bytes.NewReader(lowerLimit)).val, NewTextValue(bytes.NewReader(upperLimit)).val) + + case LESSTHANEQUAL: + lowerLimit := append(searchKey, ClusteringType) + lowerLimit = append(lowerLimit, clusteringKyes[index].getType().getVal()) + upperLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + iterTemp := storedb.NewIterator(nil, nil) + iterTemp.Seek(upperLimit) + isAtFirst = true + for (isAtFirst || iterTemp.Next()) && bytes.HasPrefix(iterTemp.Key(), upperLimit){ + isAtFirst = false + } + iter = storedb.NewIterator(&util.Range{Start: lowerLimit, Limit: iterTemp.Key()}, nil) + isAtFirst = false + + case BETWEEN: + lowerLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + index++ + upperLimit := append(searchKey,buildKey(ClusteringType, clusteringKyes[index])...) + iter = storedb.NewIterator(&util.Range{Start: lowerLimit , Limit: upperLimit}, nil) + } + } + i := 0 + for isAtFirst || iter.Next() { + isAtFirst = false + if iter.Key() == nil{ + break + } + cKeyvalues := []FleetDBValue{} + colvalues := []FleetDBValue{} + //fmt.Printf("key: %s | value: %s\n", item.Key, item.Value) + r := bytes.NewReader(iter.Key()) + p := make([]byte, 1) + _ , err := r.Read(p) + j := uint8(1) + for err != io.EOF{ + var val FleetDBValue + keyType := ColKeyType(p[0]) + r.Read(p) + dataType := FleetDBType(p[0]) + switch dataType { + case Int: + val = NewIntValue(r) + case BigInt: + val = NewBigIntValue(r) + case Float: + val = NewFloatValue(r) + case Double: + val = NewDoubleValue(r) + case Text: + val = NewTextValueFromNullTerminatedStream(r) + case Boolean: + val = NewBooleanValue(r) + } + if(keyType == PRIMARY){ + valMap[seqMap[j]] = val + } + if(keyType == CLUSTERING){ + valMap[seqMap[j]] = val + cKeyvalues = append(cKeyvalues, val) + }else if(keyType == COLUMN) { + colvalues = append(colvalues,NewTextValue(bytes.NewReader(iter.Value()))) + valMap[val.String()] = NewTextValue(bytes.NewReader(iter.Value())) + } + _ , err = r.Read(p) + j += 1 + } + isEq := equal(oldCKeyvalues, cKeyvalues) + if i != 0 && (!isEq){ + myrow := createRowFromMap(oldValMap, columns) + rows = append(rows,myrow) + oldColvalues = colvalues + oldValMap = valMap + + }else{ + oldColvalues = append(oldColvalues,colvalues...) + for k,v := range valMap { + oldValMap[k] = v + } + } + oldCKeyvalues = cKeyvalues + valMap = make(map[string]FleetDBValue) + //oldValMap = valMap + //valMap := make(map[string]FleetDBValue) + //oldValMap := make(map[string]FleetDBValue) + i++ + } + myrow := createRowFromMap(oldValMap, columns) + rows = append(rows,myrow) + return rows,err +} + +func createRowFromMap(valMap map[string]FleetDBValue, columns []string) Row{ + row := []FleetDBValue{} + for _ , colName := range columns{ + row = append(row, valMap[colName]) + } + return Row{row} +} + +func equal(val1 []FleetDBValue, val2 []FleetDBValue) bool{ + if(len(val1) != len(val2)){ + return false + } + for i, _ := range val1{ + if val1[i].String() != val2[i].String(){ + return false + } + } + return true +} \ No newline at end of file diff --git a/tablestore/db_test.go b/tablestore/db_test.go new file mode 100644 index 0000000..07a4831 --- /dev/null +++ b/tablestore/db_test.go @@ -0,0 +1,415 @@ +package tablestore + +import ( + "testing" + "github.com/google/uuid" + // "fmt" + "github.com/stretchr/testify/assert" +) + +func createSampleDB()(uuid.UUID, []FleetDBValue, []string, FleetDBTableSpec, Store){ + db := NewStore() + tableid := uuid.New() + tableMap := make(map[string][]FleetDbColumnSpec) + tableName := "crossfit_gyms" + var myText TextValue + + //Test With One Partition Key and One Clustering Key + myschema := make([]FleetDbColumnSpec, 4) + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false, 1} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true, 2} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false, 3} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false, 4} + tableMap[tableName] = myschema + + insertCommand1 := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; + rowData, myTableName := decodeInsertCommand(insertCommand1) + res := TranslateToKV(tableMap[myTableName], rowData) + data := TranslateToKV(tableMap[tableName], []FleetDBValue{TextValue{"US"}, TextValue{"Ohio"},TextValue{"Columbus"},TextValue{"Englewood Avenue"}}) + res = append(res,data...) + db.Save(tableid, res) + + columnSpecs := make(map[string]*FleetDbColumnSpec) + columnSpecs["country_code"] = &myschema[0] + columnSpecs["state_province"] = &myschema[1] + columnSpecs["city"] = &myschema[2] + columnSpecs["gym_name"] = &myschema[3] + tableSpec := FleetDBTableSpec{columnSpecs} + partitionKeys := []FleetDBValue{TextValue{"US"}} + columns := []string{"country_code","state_province", "city", "gym_name"} + return tableid,partitionKeys,columns,tableSpec,db +} + +func TestBETWEEN(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + //fmt.Println("Testing BETWEEN") + expectedVal := [][]string{{"US", "NY", "Buffalo", "University Avenue"}} + clusteringKyes := []FleetDBValue {TextValue{"ABC"}, TextValue{"OA"}} + clusteringComparators := []ComparatorType{BETWEEN} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestGREATERTHANEQUAL(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + //fmt.Println("Testing GREATERTHANEQUAL 1") + expectedVal := [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringKyes := []FleetDBValue {TextValue{"NY"}} + clusteringComparators := []ComparatorType{GREATERTHANEQUAL} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + //fmt.Println("Testing GREATERTHANEQUAL 2") + expectedVal = [][]string{{"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringKyes = []FleetDBValue {TextValue{"NYA"}} + clusteringComparators = []ComparatorType{GREATERTHANEQUAL} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 1) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestGREATERTHAN(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + //fmt.Println("Testing GREATERTHAN 1") + expectedVal := [][]string{{"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringKyes := []FleetDBValue {TextValue{"NY"}} + clusteringComparators := []ComparatorType{GREATERTHAN} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + assert.Equal(t, len(rows), 1) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + //fmt.Println("Testing GREATERTHAN 2") + expectedVal = [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringKyes = []FleetDBValue {TextValue{"ABC"}} + clusteringComparators = []ComparatorType{GREATERTHAN} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestLESSERTHANEQUAL(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + //fmt.Println("Testing LESSERTHANEQUAL 1") + clusteringKyes := []FleetDBValue {TextValue{"Ohio"}} + expectedVal := [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringComparators := []ComparatorType{LESSTHANEQUAL} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + //fmt.Println(len(rows)) + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + //fmt.Println("Testing LESSERTHANEQUAL 2") + clusteringKyes = []FleetDBValue {TextValue{"NY"}} + expectedVal = [][]string{{"US", "NY", "Buffalo", "University Avenue"}} + clusteringComparators = []ComparatorType{LESSTHANEQUAL} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 1) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestLESSERTHAN(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + //fmt.Println("Testing LESSERTHAN 1") + clusteringKyes := []FleetDBValue {TextValue{"Ohio"}} + expectedVal := [][]string{{"US", "NY", "Buffalo", "University Avenue"}} + clusteringComparators := []ComparatorType{LESSTHAN} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + assert.Equal(t, len(rows), 1) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + //fmt.Println("Testing LESSERTHAN 2") + clusteringKyes = []FleetDBValue {TextValue{"ZAB"}} + expectedVal = [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + clusteringComparators = []ComparatorType{LESSTHAN} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestCreateRowFromMap(t *testing.T) { + columns := []string{"country_code","state_province", "city", "gym_name"} + valMap := make(map[string]FleetDBValue) + valMap["country_code"] = TextValue{"US"} + valMap["state_province"] = TextValue{"NY"} + valMap["city"] = TextValue{"Buffalo"} + valMap["gym_name"] = TextValue{"University Avenue"} + + //All rows required + row := createRowFromMap(valMap, columns) + assert.Equal(t, row.Values[0], TextValue{"US"}) + assert.Equal(t, row.Values[1], TextValue{"NY"}) + assert.Equal(t, row.Values[2], TextValue{"Buffalo"}) + assert.Equal(t, row.Values[3], TextValue{"University Avenue"}) + + //Only 2 row required + columns = []string{"country_code", "gym_name"} + row = createRowFromMap(valMap, columns) + assert.Equal(t, row.Values[0], TextValue{"US"}) + assert.Equal(t, row.Values[1], TextValue{"University Avenue"}) + + //when column value is not present in database + columns = []string{"country_code", "street_name"} + row = createRowFromMap(valMap, columns) + assert.Equal(t, row.Values[0], TextValue{"US"}) + assert.Equal(t, row.Values[1], nil) + +} + +func TestGetPartition(t *testing.T) { + tableid,partitionKeys,columns,tableSpec,db := createSampleDB() + + //Retrieving only 2 columns + columns = []string{"state_province", "city"} + clusteringKyes := []FleetDBValue {} + clusteringComparators := []ComparatorType{} + expectedVal := [][]string{{"NY", "Buffalo"}, {"Ohio", "Columbus"}} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + + //Retrieving All columns + columns = []string{"country_code", "state_province", "city", "gym_name"} + clusteringKyes = []FleetDBValue {} + clusteringComparators = []ComparatorType{} + expectedVal = [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } +} + +func TestSave(t *testing.T){ + + //Test Insert + db := NewStore() + tableid := uuid.New() + tableMap := make(map[string][]FleetDbColumnSpec) + tableName := "crossfit_gyms" + var myText TextValue + + myschema := make([]FleetDbColumnSpec, 4) + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false, 1} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true, 2} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false, 3} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false, 4} + tableMap[tableName] = myschema + + insertCommand1 := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; + rowData, myTableName := decodeInsertCommand(insertCommand1) + res := TranslateToKV(tableMap[myTableName], rowData) + data := TranslateToKV(tableMap[tableName], []FleetDBValue{TextValue{"US"}, TextValue{"Ohio"},TextValue{"Columbus"},TextValue{"Englewood Avenue"}}) + res = append(res,data...) + db.Save(tableid, res) + + + columnSpecs := make(map[string]*FleetDbColumnSpec) + columnSpecs["country_code"] = &myschema[0] + columnSpecs["state_province"] = &myschema[1] + columnSpecs["city"] = &myschema[2] + columnSpecs["gym_name"] = &myschema[3] + tableSpec := FleetDBTableSpec{columnSpecs} + partitionKeys := []FleetDBValue{TextValue{"US"}} + columns := []string{"country_code", "state_province", "city", "gym_name"} + clusteringKyes := []FleetDBValue {} + clusteringComparators := []ComparatorType{} + expectedVal := [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood Avenue"}} + rows, _ := db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum := 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + + // Test Update + + data = TranslateToKV(tableMap[tableName], []FleetDBValue{TextValue{"US"}, TextValue{"Ohio"},TextValue{"Columbus"},TextValue{"Englewood New Avenue"}}) + db.Save(tableid, data) + + expectedVal = [][]string{{"US", "NY", "Buffalo", "University Avenue"}, {"US", "Ohio", "Columbus", "Englewood New Avenue"}} + rows, _ = db.GetPartition(tableid, partitionKeys, clusteringKyes, clusteringComparators, columns, tableSpec) + rowNum = 0 + assert.Equal(t, len(rows), 2) + for _, row := range rows{ + //ch := "" + index := 0 + for _, val := range row.Values{ + assert.Equal(t, val.String(), expectedVal[rowNum][index]) + if(val != nil){ + //fmt.Printf(ch + val.String()) + } + //ch = " || " + index += 1 + } + //fmt.Println() + rowNum++ + } + +} + diff --git a/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/000001.log b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..0408b763d5048c6eafaf475ddd6e6e4c5de94129 GIT binary patch literal 49 vcmb+4 literal 0 HcmV?d00001 diff --git a/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/CURRENT b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/CURRENT new file mode 100644 index 0000000..feda7d6 --- /dev/null +++ b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/CURRENT @@ -0,0 +1 @@ +MANIFEST-000000 diff --git a/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOCK b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOCK new file mode 100644 index 0000000..e69de29 diff --git a/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOG b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOG new file mode 100644 index 0000000..4a64a0a --- /dev/null +++ b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/LOG @@ -0,0 +1,6 @@ +=============== Oct 12, 2019 (EDT) =============== +15:30:07.296659 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:30:07.320460 db@open opening +15:30:07.323607 version@stat F·[] S·0B[] Sc·[] +15:30:07.338370 db@janitor F·2 G·0 +15:30:07.339100 db@open done T·17.301754ms diff --git a/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/MANIFEST-000000 b/tablestore/myDB/25431cf8-3e99-460e-be81-f784a1400873/MANIFEST-000000 new file mode 100644 index 0000000000000000000000000000000000000000..9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d GIT binary patch literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 diff --git a/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/000001.log b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..0408b763d5048c6eafaf475ddd6e6e4c5de94129 GIT binary patch literal 49 vcmb+4 literal 0 HcmV?d00001 diff --git a/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/CURRENT b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/CURRENT new file mode 100644 index 0000000..feda7d6 --- /dev/null +++ b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/CURRENT @@ -0,0 +1 @@ +MANIFEST-000000 diff --git a/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOCK b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOCK new file mode 100644 index 0000000..e69de29 diff --git a/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOG b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOG new file mode 100644 index 0000000..ea688b5 --- /dev/null +++ b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/LOG @@ -0,0 +1,6 @@ +=============== Oct 12, 2019 (EDT) =============== +15:24:06.380136 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:24:06.406744 db@open opening +15:24:06.409675 version@stat F·[] S·0B[] Sc·[] +15:24:06.426362 db@janitor F·2 G·0 +15:24:06.427099 db@open done T·19.337227ms diff --git a/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/MANIFEST-000000 b/tablestore/myDB/ad50d105-cdec-44f5-9eba-e2537944cd26/MANIFEST-000000 new file mode 100644 index 0000000000000000000000000000000000000000..9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d GIT binary patch literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 diff --git a/tablestore/row.go b/tablestore/row.go new file mode 100644 index 0000000..458b184 --- /dev/null +++ b/tablestore/row.go @@ -0,0 +1,9 @@ +package tablestore + +import ( + +) +type Row struct { + Values []FleetDBValue +} + diff --git a/tablestore/table_schema.go b/tablestore/table_schema.go index 4e2a33b..9a50e71 100644 --- a/tablestore/table_schema.go +++ b/tablestore/table_schema.go @@ -9,4 +9,5 @@ type FleetDbColumnSpec struct { coltype FleetDBValue isPartition bool isClustering bool + seqNo uint8 } diff --git a/tablestore/translatetoKV_test.go b/tablestore/translatetoKV_test.go index 55e4a78..3a44376 100644 --- a/tablestore/translatetoKV_test.go +++ b/tablestore/translatetoKV_test.go @@ -2,12 +2,11 @@ package tablestore import ( "testing" - // "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" //"fmt" ) func TestTranslateToKV(t *testing.T) { - //createCommand := "CREATE TABLE crossfit_gyms (country_code string,state_province string,city string,gym_name string,PRIMARY KEY (country_code, state_province));" tableMap := make(map[string][]FleetDbColumnSpec) tableName := "crossfit_gyms" @@ -16,77 +15,102 @@ func TestTranslateToKV(t *testing.T) { //Test With One Partition Key and One Clustering Key myschema := make([]FleetDbColumnSpec, 4) - myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} - myschema[2] = FleetDbColumnSpec{"city", myText, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} + myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false, 1} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true, 2} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false, 3} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false, 4} tableMap[tableName] = myschema insertCommand := "INSERT INTO crossfit_gyms (country_code, state_province, city, gym_name) VALUES ('US', ‘NY’, ‘Buffalo’, 'University Avenue');"; rowData, myTableName := decodeInsertCommand(insertCommand) res := TranslateToKV(tableMap[myTableName], rowData) - parseKey(res) - //fmt.Println(res[0].Key) - //fmt.Println(res[1].Key) - - //fmt.Println(string([]byte{3,5,99,105,116,121,0})) - //assert.Equal(t, "US/NY/city", string(res[0].Key)) - //assert.Equal(t, "Buffalo", string(res[0].Value)) - //assert.Equal(t, "US/NY/gym_name", string(res[1].Key)) - //assert.Equal(t, "University Avenue", string(res[1].Value)) - /* + + expectedByteKey := [][][]byte{{[]byte{1},[]byte{5},[]byte("US\000"), []byte{2},[]byte{5},[]byte("NY\000"), []byte{3},[]byte{5},[]byte("city\000")}, + {[]byte{1},[]byte{5},[]byte("US\000"), []byte{2},[]byte{5},[]byte("NY\000"), []byte{3},[]byte{5},[]byte("gym_name\000")}} + + expectedSingeDKey := [][]byte{} + for i, _:= range expectedByteKey{ + oneKey := make([]byte,0) + for j, _:= range expectedByteKey[i]{ + oneKey = append(oneKey, expectedByteKey[i][j]...) + } + expectedSingeDKey = append(expectedSingeDKey, oneKey) + } + expectedVal := []string{"Buffalo","University Avenue"} + for i ,_ := range res { + assert.Equal(t, expectedSingeDKey[i], res[i].Key) + assert.Equal(t, expectedVal[i], string(res[i].Value)) + } + + //Test With One Partition Key and Zero Clustering Key - myschema[1] = FleetDbColumnSpec{"state_province", myText, false, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, false, 2} res = TranslateToKV(tableMap[myTableName], rowData) - assert.Equal(t, "US/state_province", string(res[0].Key)) - assert.Equal(t, "NY", string(res[0].Value)) - assert.Equal(t, "US/city", string(res[1].Key)) - assert.Equal(t, "Buffalo", string(res[1].Value)) - assert.Equal(t, "US/gym_name", string(res[2].Key)) - assert.Equal(t, "University Avenue", string(res[2].Value)) + + expectedByteKey = [][][]byte{{[]byte{1},[]byte{5},[]byte("US\000"), []byte{3},[]byte{5},[]byte("state_province\000")}, + {[]byte{1},[]byte{5},[]byte("US\000"), []byte{3},[]byte{5},[]byte("city\000")}, + {[]byte{1},[]byte{5},[]byte("US\000"), []byte{3},[]byte{5},[]byte("gym_name\000")}} + + expectedSingeDKey = [][]byte{} + for i, _:= range expectedByteKey{ + oneKey := make([]byte,0) + for j, _:= range expectedByteKey[i]{ + oneKey = append(oneKey, expectedByteKey[i][j]...) + } + expectedSingeDKey = append(expectedSingeDKey, oneKey) + } + expectedVal = []string{"NY", "Buffalo","University Avenue"} + for i ,_ := range res { + assert.Equal(t, expectedSingeDKey[i], res[i].Key) + assert.Equal(t, expectedVal[i], string(res[i].Value)) + } + //All columns are either PK or CK - myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} - myschema[2] = FleetDbColumnSpec{"city", myText, false, true} - myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, true} + myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true, 2} + myschema[2] = FleetDbColumnSpec{"city", myText, false, true, 3} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, true, 4} res = TranslateToKV(tableMap[myTableName], rowData) - assert.Equal(t, "US/NY/Buffalo/University Avenue", string(res[0].Key)) - assert.Equal(t,[]byte(nil), res[0].Value) + + expectedByteKey = [][][]byte{{[]byte{1},[]byte{5},[]byte("US\000"), []byte{2},[]byte{5},[]byte("NY\000"), []byte{2},[]byte{5},[]byte("Buffalo\000"), []byte{2},[]byte{5},[]byte("University Avenue\000")}} + + expectedSingeDKey = [][]byte{} + for i, _:= range expectedByteKey{ + oneKey := make([]byte,0) + for j, _:= range expectedByteKey[i]{ + oneKey = append(oneKey, expectedByteKey[i][j]...) + } + expectedSingeDKey = append(expectedSingeDKey, oneKey) + } + for i ,_ := range res { + assert.Equal(t, expectedSingeDKey[i], res[i].Key) + assert.Equal(t,[]byte(nil), res[i].Value) + } // Composite Key - myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myText, true, false} - myschema[2] = FleetDbColumnSpec{"city", myText, false, false} - myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} - res = TranslateToKV(tableMap[myTableName], rowData) - assert.Equal(t, "US/NY/city", string(res[0].Key)) - assert.Equal(t, "Buffalo", string(res[0].Value)) - assert.Equal(t, "US/NY/gym_name", string(res[1].Key)) - assert.Equal(t, "University Avenue", string(res[1].Value)) - - //Composite Key with Clustering Key - myschema[0] = FleetDbColumnSpec{"country_code", myText, true, false} - myschema[1] = FleetDbColumnSpec{"state_province", myText, false, true} - myschema[2] = FleetDbColumnSpec{"city", myText, true, false} - myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false} + myschema[1] = FleetDbColumnSpec{"state_province", myText, true, false, 2} + myschema[2] = FleetDbColumnSpec{"city", myText, false, false, 3} + myschema[3] = FleetDbColumnSpec{"gym_name", myText, false, false, 4} res = TranslateToKV(tableMap[myTableName], rowData) - assert.Equal(t, "US/Buffalo/NY/gym_name", string(res[0].Key)) - assert.Equal(t, "University Avenue", string(res[0].Value)) - */ + + expectedByteKey = [][][]byte{{[]byte{1},[]byte{5},[]byte("US\000"), []byte{1},[]byte{5},[]byte("NY\000"), []byte{3},[]byte{5},[]byte("city\000")}, + {[]byte{1},[]byte{5},[]byte("US\000"), []byte{1},[]byte{5},[]byte("NY\000"), []byte{3},[]byte{5},[]byte("gym_name\000")}} + + expectedSingeDKey = [][]byte{} + for i, _:= range expectedByteKey{ + oneKey := make([]byte,0) + for j, _:= range expectedByteKey[i]{ + oneKey = append(oneKey, expectedByteKey[i][j]...) + } + expectedSingeDKey = append(expectedSingeDKey, oneKey) + } + expectedVal = []string{"Buffalo","University Avenue"} + for i ,_ := range res { + assert.Equal(t, expectedSingeDKey[i], res[i].Key) + assert.Equal(t, expectedVal[i], string(res[i].Value)) + } } func decodeInsertCommand(query string)([]FleetDBValue , string){ val := []FleetDBValue{TextValue{"US"}, TextValue{"NY"},TextValue{"Buffalo"},TextValue{"University Avenue"}} return val, "crossfit_gyms" } - - -/*func createSchema(query string) ([]FleetDbColumnSpec,string) { - schema := make([]FleetDbColumnSpec, 4) - schema[0] = FleetDbColumnSpec{"country_code", string, true, false} - schema[1] = FleetDbColumnSpec{"state_province", string, false, true} - schema[2] = FleetDbColumnSpec{"city", string, false, false} - schema[3] = FleetDbColumnSpec{"gym_name", string, false, false} - return schema,"crossfit_gyms" -} -*/ diff --git a/tablestore/utility.go b/tablestore/utility.go index 4ccb583..da4cfa0 100644 --- a/tablestore/utility.go +++ b/tablestore/utility.go @@ -1,10 +1,6 @@ package tablestore import ( - "fmt" - "io" - "bytes" - //"encoding/binary" ) const( @@ -45,11 +41,8 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values []FleetDBValue) []KVI cKeyCount = cKeyCount + 1 } } - //Handle Nil values - kvItems := []KVItem{} var prefix []byte - //var key []byte prefix = pKey if cKeyCount > 0{ @@ -58,9 +51,8 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values []FleetDBValue) []KVI if len(columnSpecs) == (pKeyCount + cKeyCount){ key := prefix var val []byte - kvItems[0] = KVItem{key, val} + kvItems = append(kvItems, KVItem{key, val}) }else{ - //index := 0 for i, colSpec := range columnSpecs { if !colSpec.isPartition && !colSpec.isClustering{ //fmt.Println(index) @@ -69,37 +61,15 @@ func TranslateToKV(columnSpecs []FleetDbColumnSpec, values []FleetDBValue) []KVI key = append(key,ColumnType) key = append(key, Text.getVal()) key = append(key,[]byte(colSpec.colname + nullchar)...) - val := values[i].Serialize() + var val []byte + if(values[i] != nil){ + val = values[i].Serialize() + }else { + val = nil + } kvItems = append(kvItems, KVItem{key, val}) } } } return kvItems; } - -//Incomplete Implementation -func parseKey(data []KVItem){ - for _, item := range data{ - r := bytes.NewReader(item.Key) - p := make([]byte, 1) - //var i uint8 - //fmt.Println(string(item.Key)) - _ , err := r.Read(p) - for err != io.EOF{ - r.Read(p) - //r.Read(p) - switch p[0] { - case 1: - fmt.Printf("Integer") - case 2: - fmt.Printf("BigInt") - case 5: - //fmt.Printf("Text \n") - text := NewTextValueFromNullTerminatedStream(r) - fmt.Println(text.val) - } - _ , err = r.Read(p) - } - } -} -