diff --git a/Makefile b/Makefile index 261defd0..dae01961 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ mocks-gen: mocks-rm ## Generate mocks for all the defined interfaces. mockgen -package=mockschema -source=pkg/schema/schema_client.go -destination=$(MOCKDIR)/mockschema/client.go mockgen -package=mockschemaclientbound -source=pkg/datastore/clients/schema/schemaClientBound.go -destination=$(MOCKDIR)/mockschemaclientbound/client.go mockgen -package=mockcacheclient -source=pkg/cache/cache.go -destination=$(MOCKDIR)/mockcacheclient/client.go + mockgen -package=mockcacheclient -source=pkg/cache/cacheClientBound.go -destination=$(MOCKDIR)/mockcacheclient/clientbound.go mockgen -package=mocktarget -source=pkg/datastore/target/target.go -destination=$(MOCKDIR)/mocktarget/target.go mockgen -package=mockTreeEntry -source=pkg/tree/entry.go -destination=$(MOCKDIR)/mocktreeentry/entry.go diff --git a/go.mod b/go.mod index 476efac2..23656402 100644 --- a/go.mod +++ b/go.mod @@ -24,10 +24,11 @@ require ( github.com/sdcio/cache v0.0.38 github.com/sdcio/logger v0.0.3 github.com/sdcio/schema-server v0.0.34 - github.com/sdcio/sdc-protos v0.0.49 + github.com/sdcio/sdc-protos v0.0.50 github.com/sdcio/yang-parser v0.0.12 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 @@ -81,6 +82,7 @@ require ( github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 // indirect github.com/openconfig/grpctunnel v0.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect diff --git a/go.sum b/go.sum index b978a562..cc73088d 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= github.com/sdcio/logger v0.0.3/go.mod h1:yWaOxK/G6vszjg8tKZiMqiEjlZouHsjFME4zSk+SAEA= github.com/sdcio/schema-server v0.0.34 h1:NNDOkvtUMONtBA7cVvN96F+FWGD/Do6HNqfchy9B8eI= github.com/sdcio/schema-server v0.0.34/go.mod h1:6t8HLXpqUqEJmE5yNZh29u/KZw0jlOICdNWns7zE4GE= -github.com/sdcio/sdc-protos v0.0.49 h1:GpLTDEyNnQxO5fjY7b8Y4xMP8f9kr4bbCHPJYVZkK00= -github.com/sdcio/sdc-protos v0.0.49/go.mod h1:huh1QVE023w+reU2Gt6h1f83R3lJidcFLbQje7cMY1M= +github.com/sdcio/sdc-protos v0.0.50 h1:aR6Av1QMTFNXKKxnrz8vlIXQ0y21lxQJEMut1oLd2Bg= +github.com/sdcio/sdc-protos v0.0.50/go.mod h1:huh1QVE023w+reU2Gt6h1f83R3lJidcFLbQje7cMY1M= github.com/sdcio/yang-parser v0.0.12 h1:RSSeqfAOIsJx5Lno5u4/ezyOmQYUduQ22rBfU/mtpJ4= github.com/sdcio/yang-parser v0.0.12/go.mod h1:CBqn3Miq85qmFVGHxHXHLluXkaIOsTzV06IM4DW6+D4= github.com/sirikothe/gotextfsm v1.0.1-0.20200816110946-6aa2cfd355e4 h1:FHUL2HofYJuslFOQdy/JjjP36zxqIpd/dcoiwLMIs7k= diff --git a/mocks/mockcacheclient/clientbound.go b/mocks/mockcacheclient/clientbound.go new file mode 100644 index 00000000..4682d5b6 --- /dev/null +++ b/mocks/mockcacheclient/clientbound.go @@ -0,0 +1,183 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/cache/cacheClientBound.go +// +// Generated by this command: +// +// mockgen -package=mockcacheclient -source=pkg/cache/cacheClientBound.go -destination=./mocks/mockcacheclient/clientbound.go +// + +// Package mockcacheclient is a generated GoMock package. +package mockcacheclient + +import ( + context "context" + reflect "reflect" + + tree_persist "github.com/sdcio/sdc-protos/tree_persist" + gomock "go.uber.org/mock/gomock" +) + +// MockCacheClientBound is a mock of CacheClientBound interface. +type MockCacheClientBound struct { + ctrl *gomock.Controller + recorder *MockCacheClientBoundMockRecorder + isgomock struct{} +} + +// MockCacheClientBoundMockRecorder is the mock recorder for MockCacheClientBound. +type MockCacheClientBoundMockRecorder struct { + mock *MockCacheClientBound +} + +// NewMockCacheClientBound creates a new mock instance. +func NewMockCacheClientBound(ctrl *gomock.Controller) *MockCacheClientBound { + mock := &MockCacheClientBound{ctrl: ctrl} + mock.recorder = &MockCacheClientBoundMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCacheClientBound) EXPECT() *MockCacheClientBoundMockRecorder { + return m.recorder +} + +// InstanceClose mocks base method. +func (m *MockCacheClientBound) InstanceClose(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceClose", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceClose indicates an expected call of InstanceClose. +func (mr *MockCacheClientBoundMockRecorder) InstanceClose(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceClose", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceClose), ctx) +} + +// InstanceCreate mocks base method. +func (m *MockCacheClientBound) InstanceCreate(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceCreate", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceCreate indicates an expected call of InstanceCreate. +func (mr *MockCacheClientBoundMockRecorder) InstanceCreate(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceCreate", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceCreate), ctx) +} + +// InstanceDelete mocks base method. +func (m *MockCacheClientBound) InstanceDelete(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceDelete", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceDelete indicates an expected call of InstanceDelete. +func (mr *MockCacheClientBoundMockRecorder) InstanceDelete(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceDelete", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceDelete), ctx) +} + +// InstanceExists mocks base method. +func (m *MockCacheClientBound) InstanceExists(ctx context.Context) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceExists", ctx) + ret0, _ := ret[0].(bool) + return ret0 +} + +// InstanceExists indicates an expected call of InstanceExists. +func (mr *MockCacheClientBoundMockRecorder) InstanceExists(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceExists", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceExists), ctx) +} + +// IntentDelete mocks base method. +func (m *MockCacheClientBound) IntentDelete(ctx context.Context, intentName string, IgnoreNonExisting bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentDelete", ctx, intentName, IgnoreNonExisting) + ret0, _ := ret[0].(error) + return ret0 +} + +// IntentDelete indicates an expected call of IntentDelete. +func (mr *MockCacheClientBoundMockRecorder) IntentDelete(ctx, intentName, IgnoreNonExisting any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentDelete", reflect.TypeOf((*MockCacheClientBound)(nil).IntentDelete), ctx, intentName, IgnoreNonExisting) +} + +// IntentExists mocks base method. +func (m *MockCacheClientBound) IntentExists(ctx context.Context, intentName string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentExists", ctx, intentName) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentExists indicates an expected call of IntentExists. +func (mr *MockCacheClientBoundMockRecorder) IntentExists(ctx, intentName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentExists", reflect.TypeOf((*MockCacheClientBound)(nil).IntentExists), ctx, intentName) +} + +// IntentGet mocks base method. +func (m *MockCacheClientBound) IntentGet(ctx context.Context, intentName string) (*tree_persist.Intent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentGet", ctx, intentName) + ret0, _ := ret[0].(*tree_persist.Intent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentGet indicates an expected call of IntentGet. +func (mr *MockCacheClientBoundMockRecorder) IntentGet(ctx, intentName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentGet", reflect.TypeOf((*MockCacheClientBound)(nil).IntentGet), ctx, intentName) +} + +// IntentGetAll mocks base method. +func (m *MockCacheClientBound) IntentGetAll(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "IntentGetAll", ctx, excludeIntentNames, intentChan, errChan) +} + +// IntentGetAll indicates an expected call of IntentGetAll. +func (mr *MockCacheClientBoundMockRecorder) IntentGetAll(ctx, excludeIntentNames, intentChan, errChan any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentGetAll", reflect.TypeOf((*MockCacheClientBound)(nil).IntentGetAll), ctx, excludeIntentNames, intentChan, errChan) +} + +// IntentModify mocks base method. +func (m *MockCacheClientBound) IntentModify(ctx context.Context, intent *tree_persist.Intent) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentModify", ctx, intent) + ret0, _ := ret[0].(error) + return ret0 +} + +// IntentModify indicates an expected call of IntentModify. +func (mr *MockCacheClientBoundMockRecorder) IntentModify(ctx, intent any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentModify", reflect.TypeOf((*MockCacheClientBound)(nil).IntentModify), ctx, intent) +} + +// IntentsList mocks base method. +func (m *MockCacheClientBound) IntentsList(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentsList", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentsList indicates an expected call of IntentsList. +func (mr *MockCacheClientBoundMockRecorder) IntentsList(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentsList", reflect.TypeOf((*MockCacheClientBound)(nil).IntentsList), ctx) +} diff --git a/mocks/mockschema/client.go b/mocks/mockschema/client.go index 437e77c8..3cb356b1 100644 --- a/mocks/mockschema/client.go +++ b/mocks/mockschema/client.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" grpc "google.golang.org/grpc" ) @@ -43,14 +43,14 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // CreateSchema mocks base method. -func (m *MockClient) CreateSchema(ctx context.Context, in *schema_server.CreateSchemaRequest, opts ...grpc.CallOption) (*schema_server.CreateSchemaResponse, error) { +func (m *MockClient) CreateSchema(ctx context.Context, in *sdcpb.CreateSchemaRequest, opts ...grpc.CallOption) (*sdcpb.CreateSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CreateSchema", varargs...) - ret0, _ := ret[0].(*schema_server.CreateSchemaResponse) + ret0, _ := ret[0].(*sdcpb.CreateSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -63,14 +63,14 @@ func (mr *MockClientMockRecorder) CreateSchema(ctx, in any, opts ...any) *gomock } // DeleteSchema mocks base method. -func (m *MockClient) DeleteSchema(ctx context.Context, in *schema_server.DeleteSchemaRequest, opts ...grpc.CallOption) (*schema_server.DeleteSchemaResponse, error) { +func (m *MockClient) DeleteSchema(ctx context.Context, in *sdcpb.DeleteSchemaRequest, opts ...grpc.CallOption) (*sdcpb.DeleteSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "DeleteSchema", varargs...) - ret0, _ := ret[0].(*schema_server.DeleteSchemaResponse) + ret0, _ := ret[0].(*sdcpb.DeleteSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -83,14 +83,14 @@ func (mr *MockClientMockRecorder) DeleteSchema(ctx, in any, opts ...any) *gomock } // ExpandPath mocks base method. -func (m *MockClient) ExpandPath(ctx context.Context, in *schema_server.ExpandPathRequest, opts ...grpc.CallOption) (*schema_server.ExpandPathResponse, error) { +func (m *MockClient) ExpandPath(ctx context.Context, in *sdcpb.ExpandPathRequest, opts ...grpc.CallOption) (*sdcpb.ExpandPathResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ExpandPath", varargs...) - ret0, _ := ret[0].(*schema_server.ExpandPathResponse) + ret0, _ := ret[0].(*sdcpb.ExpandPathResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -103,14 +103,14 @@ func (mr *MockClientMockRecorder) ExpandPath(ctx, in any, opts ...any) *gomock.C } // GetSchema mocks base method. -func (m *MockClient) GetSchema(ctx context.Context, in *schema_server.GetSchemaRequest, opts ...grpc.CallOption) (*schema_server.GetSchemaResponse, error) { +func (m *MockClient) GetSchema(ctx context.Context, in *sdcpb.GetSchemaRequest, opts ...grpc.CallOption) (*sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchema", varargs...) - ret0, _ := ret[0].(*schema_server.GetSchemaResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -123,14 +123,14 @@ func (mr *MockClientMockRecorder) GetSchema(ctx, in any, opts ...any) *gomock.Ca } // GetSchemaDetails mocks base method. -func (m *MockClient) GetSchemaDetails(ctx context.Context, in *schema_server.GetSchemaDetailsRequest, opts ...grpc.CallOption) (*schema_server.GetSchemaDetailsResponse, error) { +func (m *MockClient) GetSchemaDetails(ctx context.Context, in *sdcpb.GetSchemaDetailsRequest, opts ...grpc.CallOption) (*sdcpb.GetSchemaDetailsResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchemaDetails", varargs...) - ret0, _ := ret[0].(*schema_server.GetSchemaDetailsResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaDetailsResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -143,14 +143,14 @@ func (mr *MockClientMockRecorder) GetSchemaDetails(ctx, in any, opts ...any) *go } // GetSchemaElements mocks base method. -func (m *MockClient) GetSchemaElements(ctx context.Context, req *schema_server.GetSchemaRequest, opts ...grpc.CallOption) (chan *schema_server.SchemaElem, error) { +func (m *MockClient) GetSchemaElements(ctx context.Context, req *sdcpb.GetSchemaRequest, opts ...grpc.CallOption) (chan *sdcpb.SchemaElem, error) { m.ctrl.T.Helper() varargs := []any{ctx, req} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchemaElements", varargs...) - ret0, _ := ret[0].(chan *schema_server.SchemaElem) + ret0, _ := ret[0].(chan *sdcpb.SchemaElem) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -163,14 +163,14 @@ func (mr *MockClientMockRecorder) GetSchemaElements(ctx, req any, opts ...any) * } // ListSchema mocks base method. -func (m *MockClient) ListSchema(ctx context.Context, in *schema_server.ListSchemaRequest, opts ...grpc.CallOption) (*schema_server.ListSchemaResponse, error) { +func (m *MockClient) ListSchema(ctx context.Context, in *sdcpb.ListSchemaRequest, opts ...grpc.CallOption) (*sdcpb.ListSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ListSchema", varargs...) - ret0, _ := ret[0].(*schema_server.ListSchemaResponse) + ret0, _ := ret[0].(*sdcpb.ListSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -183,14 +183,14 @@ func (mr *MockClientMockRecorder) ListSchema(ctx, in any, opts ...any) *gomock.C } // ReloadSchema mocks base method. -func (m *MockClient) ReloadSchema(ctx context.Context, in *schema_server.ReloadSchemaRequest, opts ...grpc.CallOption) (*schema_server.ReloadSchemaResponse, error) { +func (m *MockClient) ReloadSchema(ctx context.Context, in *sdcpb.ReloadSchemaRequest, opts ...grpc.CallOption) (*sdcpb.ReloadSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ReloadSchema", varargs...) - ret0, _ := ret[0].(*schema_server.ReloadSchemaResponse) + ret0, _ := ret[0].(*sdcpb.ReloadSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -203,14 +203,14 @@ func (mr *MockClientMockRecorder) ReloadSchema(ctx, in any, opts ...any) *gomock } // ToPath mocks base method. -func (m *MockClient) ToPath(ctx context.Context, in *schema_server.ToPathRequest, opts ...grpc.CallOption) (*schema_server.ToPathResponse, error) { +func (m *MockClient) ToPath(ctx context.Context, in *sdcpb.ToPathRequest, opts ...grpc.CallOption) (*sdcpb.ToPathResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ToPath", varargs...) - ret0, _ := ret[0].(*schema_server.ToPathResponse) + ret0, _ := ret[0].(*sdcpb.ToPathResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -223,14 +223,14 @@ func (mr *MockClientMockRecorder) ToPath(ctx, in any, opts ...any) *gomock.Call } // UploadSchema mocks base method. -func (m *MockClient) UploadSchema(ctx context.Context, opts ...grpc.CallOption) (schema_server.SchemaServer_UploadSchemaClient, error) { +func (m *MockClient) UploadSchema(ctx context.Context, opts ...grpc.CallOption) (sdcpb.SchemaServer_UploadSchemaClient, error) { m.ctrl.T.Helper() varargs := []any{ctx} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "UploadSchema", varargs...) - ret0, _ := ret[0].(schema_server.SchemaServer_UploadSchemaClient) + ret0, _ := ret[0].(sdcpb.SchemaServer_UploadSchemaClient) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mockschemaclientbound/client.go b/mocks/mockschemaclientbound/client.go index 7a5ac461..7465f246 100644 --- a/mocks/mockschemaclientbound/client.go +++ b/mocks/mockschemaclientbound/client.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" ) @@ -42,25 +42,25 @@ func (m *MockSchemaClientBound) EXPECT() *MockSchemaClientBoundMockRecorder { } // GetSchemaElements mocks base method. -func (m *MockSchemaClientBound) GetSchemaElements(ctx context.Context, p *schema_server.Path, done chan struct{}) (chan *schema_server.GetSchemaResponse, error) { +func (m *MockSchemaClientBound) GetSchemaElements(ctx context.Context, path *sdcpb.Path, done chan struct{}) (chan *sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSchemaElements", ctx, p, done) - ret0, _ := ret[0].(chan *schema_server.GetSchemaResponse) + ret := m.ctrl.Call(m, "GetSchemaElements", ctx, path, done) + ret0, _ := ret[0].(chan *sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } // GetSchemaElements indicates an expected call of GetSchemaElements. -func (mr *MockSchemaClientBoundMockRecorder) GetSchemaElements(ctx, p, done any) *gomock.Call { +func (mr *MockSchemaClientBoundMockRecorder) GetSchemaElements(ctx, path, done any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaElements", reflect.TypeOf((*MockSchemaClientBound)(nil).GetSchemaElements), ctx, p, done) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaElements", reflect.TypeOf((*MockSchemaClientBound)(nil).GetSchemaElements), ctx, path, done) } // GetSchemaSdcpbPath mocks base method. -func (m *MockSchemaClientBound) GetSchemaSdcpbPath(ctx context.Context, path *schema_server.Path) (*schema_server.GetSchemaResponse, error) { +func (m *MockSchemaClientBound) GetSchemaSdcpbPath(ctx context.Context, path *sdcpb.Path) (*sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSchemaSdcpbPath", ctx, path) - ret0, _ := ret[0].(*schema_server.GetSchemaResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mocktarget/target.go b/mocks/mocktarget/target.go index 6f7ffbeb..1fccdd01 100644 --- a/mocks/mocktarget/target.go +++ b/mocks/mocktarget/target.go @@ -15,7 +15,7 @@ import ( config "github.com/sdcio/data-server/pkg/config" types "github.com/sdcio/data-server/pkg/datastore/target/types" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" ) @@ -63,24 +63,24 @@ func (mr *MockTargetMockRecorder) AddSyncs(ctx any, sps ...any) *gomock.Call { } // Close mocks base method. -func (m *MockTarget) Close() error { +func (m *MockTarget) Close(ctx context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") + ret := m.ctrl.Call(m, "Close", ctx) ret0, _ := ret[0].(error) return ret0 } // Close indicates an expected call of Close. -func (mr *MockTargetMockRecorder) Close() *gomock.Call { +func (mr *MockTargetMockRecorder) Close(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTarget)(nil).Close)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTarget)(nil).Close), ctx) } // Get mocks base method. -func (m *MockTarget) Get(ctx context.Context, req *schema_server.GetDataRequest) (*schema_server.GetDataResponse, error) { +func (m *MockTarget) Get(ctx context.Context, req *sdcpb.GetDataRequest) (*sdcpb.GetDataResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", ctx, req) - ret0, _ := ret[0].(*schema_server.GetDataResponse) + ret0, _ := ret[0].(*sdcpb.GetDataResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -92,10 +92,10 @@ func (mr *MockTargetMockRecorder) Get(ctx, req any) *gomock.Call { } // Set mocks base method. -func (m *MockTarget) Set(ctx context.Context, source types.TargetSource) (*schema_server.SetDataResponse, error) { +func (m *MockTarget) Set(ctx context.Context, source types.TargetSource) (*sdcpb.SetDataResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Set", ctx, source) - ret0, _ := ret[0].(*schema_server.SetDataResponse) + ret0, _ := ret[0].(*sdcpb.SetDataResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mocktreeentry/entry.go b/mocks/mocktreeentry/entry.go index ee8bec50..ff0b493b 100644 --- a/mocks/mocktreeentry/entry.go +++ b/mocks/mocktreeentry/entry.go @@ -16,9 +16,8 @@ import ( etree "github.com/beevik/etree" config "github.com/sdcio/data-server/pkg/config" tree "github.com/sdcio/data-server/pkg/tree" - importer "github.com/sdcio/data-server/pkg/tree/importer" types "github.com/sdcio/data-server/pkg/tree/types" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" tree_persist "github.com/sdcio/sdc-protos/tree_persist" gomock "go.uber.org/mock/gomock" ) @@ -47,8 +46,22 @@ func (m *MockEntry) EXPECT() *MockEntryMockRecorder { return m.recorder } +// AddChild mocks base method. +func (m *MockEntry) AddChild(arg0 context.Context, arg1 tree.Entry) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChild", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChild indicates an expected call of AddChild. +func (mr *MockEntryMockRecorder) AddChild(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChild", reflect.TypeOf((*MockEntry)(nil).AddChild), arg0, arg1) +} + // AddUpdateRecursive mocks base method. -func (m *MockEntry) AddUpdateRecursive(ctx context.Context, relativePath *schema_server.Path, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { +func (m *MockEntry) AddUpdateRecursive(ctx context.Context, relativePath *sdcpb.Path, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddUpdateRecursive", ctx, relativePath, u, flags) ret0, _ := ret[0].(tree.Entry) @@ -63,7 +76,7 @@ func (mr *MockEntryMockRecorder) AddUpdateRecursive(ctx, relativePath, u, flags } // BreadthSearch mocks base method. -func (m *MockEntry) BreadthSearch(ctx context.Context, path *schema_server.Path) ([]tree.Entry, error) { +func (m *MockEntry) BreadthSearch(ctx context.Context, path *sdcpb.Path) ([]tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BreadthSearch", ctx, path) ret0, _ := ret[0].([]tree.Entry) @@ -77,6 +90,20 @@ func (mr *MockEntryMockRecorder) BreadthSearch(ctx, path any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BreadthSearch", reflect.TypeOf((*MockEntry)(nil).BreadthSearch), ctx, path) } +// CanDeleteBranch mocks base method. +func (m *MockEntry) CanDeleteBranch(keepDefault bool) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanDeleteBranch", keepDefault) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanDeleteBranch indicates an expected call of CanDeleteBranch. +func (mr *MockEntryMockRecorder) CanDeleteBranch(keepDefault any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanDeleteBranch", reflect.TypeOf((*MockEntry)(nil).CanDeleteBranch), keepDefault) +} + // DeepCopy mocks base method. func (m *MockEntry) DeepCopy(tc *tree.TreeContext, parent tree.Entry) (tree.Entry, error) { m.ctrl.T.Helper() @@ -93,7 +120,7 @@ func (mr *MockEntryMockRecorder) DeepCopy(tc, parent any) *gomock.Call { } // DeleteBranch mocks base method. -func (m *MockEntry) DeleteBranch(ctx context.Context, path *schema_server.Path, owner string) error { +func (m *MockEntry) DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteBranch", ctx, path, owner) ret0, _ := ret[0].(error) @@ -106,6 +133,18 @@ func (mr *MockEntryMockRecorder) DeleteBranch(ctx, path, owner any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBranch", reflect.TypeOf((*MockEntry)(nil).DeleteBranch), ctx, path, owner) } +// DeleteCanDeleteChilds mocks base method. +func (m *MockEntry) DeleteCanDeleteChilds(keepDefault bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "DeleteCanDeleteChilds", keepDefault) +} + +// DeleteCanDeleteChilds indicates an expected call of DeleteCanDeleteChilds. +func (mr *MockEntryMockRecorder) DeleteCanDeleteChilds(keepDefault any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCanDeleteChilds", reflect.TypeOf((*MockEntry)(nil).DeleteCanDeleteChilds), keepDefault) +} + // FilterChilds mocks base method. func (m *MockEntry) FilterChilds(keys map[string]string) ([]tree.Entry, error) { m.ctrl.T.Helper() @@ -149,8 +188,23 @@ func (mr *MockEntryMockRecorder) GetByOwner(owner, result any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByOwner", reflect.TypeOf((*MockEntry)(nil).GetByOwner), owner, result) } +// GetChild mocks base method. +func (m *MockEntry) GetChild(name string) (tree.Entry, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChild", name) + ret0, _ := ret[0].(tree.Entry) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// GetChild indicates an expected call of GetChild. +func (mr *MockEntryMockRecorder) GetChild(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChild", reflect.TypeOf((*MockEntry)(nil).GetChild), name) +} + // GetChilds mocks base method. -func (m *MockEntry) GetChilds(arg0 tree.DescendMethod) tree.EntryMap { +func (m *MockEntry) GetChilds(arg0 types.DescendMethod) tree.EntryMap { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetChilds", arg0) ret0, _ := ret[0].(tree.EntryMap) @@ -179,15 +233,15 @@ func (mr *MockEntryMockRecorder) GetDeletes(entries, aggregatePaths any) *gomock } // GetDeviations mocks base method. -func (m *MockEntry) GetDeviations(ch chan<- *types.DeviationEntry, activeCase bool) { +func (m *MockEntry) GetDeviations(ctx context.Context, ch chan<- *types.DeviationEntry, activeCase bool) { m.ctrl.T.Helper() - m.ctrl.Call(m, "GetDeviations", ch, activeCase) + m.ctrl.Call(m, "GetDeviations", ctx, ch, activeCase) } // GetDeviations indicates an expected call of GetDeviations. -func (mr *MockEntryMockRecorder) GetDeviations(ch, activeCase any) *gomock.Call { +func (mr *MockEntryMockRecorder) GetDeviations(ctx, ch, activeCase any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviations", reflect.TypeOf((*MockEntry)(nil).GetDeviations), ch, activeCase) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviations", reflect.TypeOf((*MockEntry)(nil).GetDeviations), ctx, ch, activeCase) } // GetFirstAncestorWithSchema mocks base method. @@ -305,10 +359,10 @@ func (mr *MockEntryMockRecorder) GetRootBasedEntryChain() *gomock.Call { } // GetSchema mocks base method. -func (m *MockEntry) GetSchema() *schema_server.SchemaElem { +func (m *MockEntry) GetSchema() *sdcpb.SchemaElem { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSchema") - ret0, _ := ret[0].(*schema_server.SchemaElem) + ret0, _ := ret[0].(*sdcpb.SchemaElem) return ret0 } @@ -332,32 +386,32 @@ func (mr *MockEntryMockRecorder) GetSchemaKeys() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaKeys", reflect.TypeOf((*MockEntry)(nil).GetSchemaKeys)) } -// HoldsLeafvariants mocks base method. -func (m *MockEntry) HoldsLeafvariants() bool { +// GetTreeContext mocks base method. +func (m *MockEntry) GetTreeContext() *tree.TreeContext { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HoldsLeafvariants") - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "GetTreeContext") + ret0, _ := ret[0].(*tree.TreeContext) return ret0 } -// HoldsLeafvariants indicates an expected call of HoldsLeafvariants. -func (mr *MockEntryMockRecorder) HoldsLeafvariants() *gomock.Call { +// GetTreeContext indicates an expected call of GetTreeContext. +func (mr *MockEntryMockRecorder) GetTreeContext() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HoldsLeafvariants", reflect.TypeOf((*MockEntry)(nil).HoldsLeafvariants)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTreeContext", reflect.TypeOf((*MockEntry)(nil).GetTreeContext)) } -// ImportConfig mocks base method. -func (m *MockEntry) ImportConfig(ctx context.Context, arg1 importer.ImportConfigAdapterElement, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error { +// HoldsLeafvariants mocks base method. +func (m *MockEntry) HoldsLeafvariants() bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ImportConfig", ctx, arg1, intentName, intentPrio, flags) - ret0, _ := ret[0].(error) + ret := m.ctrl.Call(m, "HoldsLeafvariants") + ret0, _ := ret[0].(bool) return ret0 } -// ImportConfig indicates an expected call of ImportConfig. -func (mr *MockEntryMockRecorder) ImportConfig(ctx, arg1, intentName, intentPrio, flags any) *gomock.Call { +// HoldsLeafvariants indicates an expected call of HoldsLeafvariants. +func (mr *MockEntryMockRecorder) HoldsLeafvariants() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportConfig", reflect.TypeOf((*MockEntry)(nil).ImportConfig), ctx, arg1, intentName, intentPrio, flags) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HoldsLeafvariants", reflect.TypeOf((*MockEntry)(nil).HoldsLeafvariants)) } // IsRoot mocks base method. @@ -390,7 +444,7 @@ func (mr *MockEntryMockRecorder) NavigateLeafRef(ctx any) *gomock.Call { } // NavigateSdcpbPath mocks base method. -func (m *MockEntry) NavigateSdcpbPath(ctx context.Context, path *schema_server.Path) (tree.Entry, error) { +func (m *MockEntry) NavigateSdcpbPath(ctx context.Context, path *sdcpb.Path) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NavigateSdcpbPath", ctx, path) ret0, _ := ret[0].(tree.Entry) @@ -418,11 +472,25 @@ func (mr *MockEntryMockRecorder) PathName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PathName", reflect.TypeOf((*MockEntry)(nil).PathName)) } +// RemainsToExist mocks base method. +func (m *MockEntry) RemainsToExist() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemainsToExist") + ret0, _ := ret[0].(bool) + return ret0 +} + +// RemainsToExist indicates an expected call of RemainsToExist. +func (mr *MockEntryMockRecorder) RemainsToExist() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemainsToExist", reflect.TypeOf((*MockEntry)(nil).RemainsToExist)) +} + // SdcpbPath mocks base method. -func (m *MockEntry) SdcpbPath() *schema_server.Path { +func (m *MockEntry) SdcpbPath() *sdcpb.Path { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SdcpbPath") - ret0, _ := ret[0].(*schema_server.Path) + ret0, _ := ret[0].(*sdcpb.Path) return ret0 } @@ -506,44 +574,31 @@ func (mr *MockEntryMockRecorder) TreeExport(owner any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TreeExport", reflect.TypeOf((*MockEntry)(nil).TreeExport), owner) } -// Validate mocks base method. -func (m *MockEntry) Validate(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { +// ValidateLevel mocks base method. +func (m *MockEntry) ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Validate", ctx, resultChan, stats, vCfg) + m.ctrl.Call(m, "ValidateLevel", ctx, resultChan, stats, vCfg) } -// Validate indicates an expected call of Validate. -func (mr *MockEntryMockRecorder) Validate(ctx, resultChan, stats, vCfg any) *gomock.Call { +// ValidateLevel indicates an expected call of ValidateLevel. +func (mr *MockEntryMockRecorder) ValidateLevel(ctx, resultChan, stats, vCfg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Validate", reflect.TypeOf((*MockEntry)(nil).Validate), ctx, resultChan, stats, vCfg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateLevel", reflect.TypeOf((*MockEntry)(nil).ValidateLevel), ctx, resultChan, stats, vCfg) } -// Walk mocks base method. -func (m *MockEntry) Walk(ctx context.Context, v tree.EntryVisitor) error { +// addUpdateRecursiveInternal mocks base method. +func (m *MockEntry) addUpdateRecursiveInternal(ctx context.Context, path *sdcpb.Path, idx int, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Walk", ctx, v) - ret0, _ := ret[0].(error) - return ret0 -} - -// Walk indicates an expected call of Walk. -func (mr *MockEntryMockRecorder) Walk(ctx, v any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockEntry)(nil).Walk), ctx, v) -} - -// addChild mocks base method. -func (m *MockEntry) addChild(arg0 context.Context, arg1 tree.Entry) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "addChild", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "addUpdateRecursiveInternal", ctx, path, idx, u, flags) + ret0, _ := ret[0].(tree.Entry) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// addChild indicates an expected call of addChild. -func (mr *MockEntryMockRecorder) addChild(arg0, arg1 any) *gomock.Call { +// addUpdateRecursiveInternal indicates an expected call of addUpdateRecursiveInternal. +func (mr *MockEntryMockRecorder) addUpdateRecursiveInternal(ctx, path, idx, u, flags any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addChild", reflect.TypeOf((*MockEntry)(nil).addChild), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addUpdateRecursiveInternal", reflect.TypeOf((*MockEntry)(nil).addUpdateRecursiveInternal), ctx, path, idx, u, flags) } // canDelete mocks base method. @@ -560,32 +615,6 @@ func (mr *MockEntryMockRecorder) canDelete() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "canDelete", reflect.TypeOf((*MockEntry)(nil).canDelete)) } -// canDeleteBranch mocks base method. -func (m *MockEntry) canDeleteBranch(keepDefault bool) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "canDeleteBranch", keepDefault) - ret0, _ := ret[0].(bool) - return ret0 -} - -// canDeleteBranch indicates an expected call of canDeleteBranch. -func (mr *MockEntryMockRecorder) canDeleteBranch(keepDefault any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "canDeleteBranch", reflect.TypeOf((*MockEntry)(nil).canDeleteBranch), keepDefault) -} - -// deleteCanDeleteChilds mocks base method. -func (m *MockEntry) deleteCanDeleteChilds(keepDefault bool) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "deleteCanDeleteChilds", keepDefault) -} - -// deleteCanDeleteChilds indicates an expected call of deleteCanDeleteChilds. -func (mr *MockEntryMockRecorder) deleteCanDeleteChilds(keepDefault any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "deleteCanDeleteChilds", reflect.TypeOf((*MockEntry)(nil).deleteCanDeleteChilds), keepDefault) -} - // getHighestPrecedenceLeafValue mocks base method. func (m *MockEntry) getHighestPrecedenceLeafValue(arg0 context.Context) (*tree.LeafEntry, error) { m.ctrl.T.Helper() @@ -616,7 +645,7 @@ func (mr *MockEntryMockRecorder) getHighestPrecedenceValueOfBranch(filter any) * } // getOrCreateChilds mocks base method. -func (m *MockEntry) getOrCreateChilds(ctx context.Context, path *schema_server.Path) (tree.Entry, error) { +func (m *MockEntry) getOrCreateChilds(ctx context.Context, path *sdcpb.Path) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "getOrCreateChilds", ctx, path) ret0, _ := ret[0].(tree.Entry) @@ -630,20 +659,6 @@ func (mr *MockEntryMockRecorder) getOrCreateChilds(ctx, path any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getOrCreateChilds", reflect.TypeOf((*MockEntry)(nil).getOrCreateChilds), ctx, path) } -// remainsToExist mocks base method. -func (m *MockEntry) remainsToExist() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "remainsToExist") - ret0, _ := ret[0].(bool) - return ret0 -} - -// remainsToExist indicates an expected call of remainsToExist. -func (mr *MockEntryMockRecorder) remainsToExist() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remainsToExist", reflect.TypeOf((*MockEntry)(nil).remainsToExist)) -} - // shouldDelete mocks base method. func (m *MockEntry) shouldDelete() bool { m.ctrl.T.Helper() @@ -712,70 +727,6 @@ func (mr *MockEntryMockRecorder) validateMandatoryWithKeys(ctx, level, attribute return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "validateMandatoryWithKeys", reflect.TypeOf((*MockEntry)(nil).validateMandatoryWithKeys), ctx, level, attributes, choiceName, resultChan) } -// MockEntryVisitor is a mock of EntryVisitor interface. -type MockEntryVisitor struct { - ctrl *gomock.Controller - recorder *MockEntryVisitorMockRecorder - isgomock struct{} -} - -// MockEntryVisitorMockRecorder is the mock recorder for MockEntryVisitor. -type MockEntryVisitorMockRecorder struct { - mock *MockEntryVisitor -} - -// NewMockEntryVisitor creates a new mock instance. -func NewMockEntryVisitor(ctrl *gomock.Controller) *MockEntryVisitor { - mock := &MockEntryVisitor{ctrl: ctrl} - mock.recorder = &MockEntryVisitorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockEntryVisitor) EXPECT() *MockEntryVisitorMockRecorder { - return m.recorder -} - -// DescendMethod mocks base method. -func (m *MockEntryVisitor) DescendMethod() tree.DescendMethod { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescendMethod") - ret0, _ := ret[0].(tree.DescendMethod) - return ret0 -} - -// DescendMethod indicates an expected call of DescendMethod. -func (mr *MockEntryVisitorMockRecorder) DescendMethod() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescendMethod", reflect.TypeOf((*MockEntryVisitor)(nil).DescendMethod)) -} - -// Up mocks base method. -func (m *MockEntryVisitor) Up() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Up") -} - -// Up indicates an expected call of Up. -func (mr *MockEntryVisitorMockRecorder) Up() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Up", reflect.TypeOf((*MockEntryVisitor)(nil).Up)) -} - -// Visit mocks base method. -func (m *MockEntryVisitor) Visit(ctx context.Context, e tree.Entry) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Visit", ctx, e) - ret0, _ := ret[0].(error) - return ret0 -} - -// Visit indicates an expected call of Visit. -func (mr *MockEntryVisitorMockRecorder) Visit(ctx, e any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Visit", reflect.TypeOf((*MockEntryVisitor)(nil).Visit), ctx, e) -} - // MockLeafVariantEntry is a mock of LeafVariantEntry interface. type MockLeafVariantEntry struct { ctrl *gomock.Controller @@ -866,6 +817,18 @@ func (m *MockLeafVariantEntries) EXPECT() *MockLeafVariantEntriesMockRecorder { return m.recorder } +// Add mocks base method. +func (m *MockLeafVariantEntries) Add(l *tree.LeafEntry) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Add", l) +} + +// Add indicates an expected call of Add. +func (mr *MockLeafVariantEntriesMockRecorder) Add(l any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockLeafVariantEntries)(nil).Add), l) +} + // AddExplicitDeleteEntry mocks base method. func (m *MockLeafVariantEntries) AddExplicitDeleteEntry(owner string, priority int32) *tree.LeafEntry { m.ctrl.T.Helper() @@ -880,6 +843,18 @@ func (mr *MockLeafVariantEntriesMockRecorder) AddExplicitDeleteEntry(owner, prio return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddExplicitDeleteEntry", reflect.TypeOf((*MockLeafVariantEntries)(nil).AddExplicitDeleteEntry), owner, priority) } +// AddWithStats mocks base method. +func (m *MockLeafVariantEntries) AddWithStats(l *tree.LeafEntry, stats *types.ImportStats) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddWithStats", l, stats) +} + +// AddWithStats indicates an expected call of AddWithStats. +func (mr *MockLeafVariantEntriesMockRecorder) AddWithStats(l, stats any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWithStats", reflect.TypeOf((*MockLeafVariantEntries)(nil).AddWithStats), l, stats) +} + // DeleteByOwner mocks base method. func (m *MockLeafVariantEntries) DeleteByOwner(owner string) *tree.LeafEntry { m.ctrl.T.Helper() @@ -936,6 +911,20 @@ func (mr *MockLeafVariantEntriesMockRecorder) GetRunning() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunning", reflect.TypeOf((*MockLeafVariantEntries)(nil).GetRunning)) } +// Length mocks base method. +func (m *MockLeafVariantEntries) Length() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Length") + ret0, _ := ret[0].(int) + return ret0 +} + +// Length indicates an expected call of Length. +func (mr *MockLeafVariantEntriesMockRecorder) Length() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Length", reflect.TypeOf((*MockLeafVariantEntries)(nil).Length)) +} + // MarkOwnerForDeletion mocks base method. func (m *MockLeafVariantEntries) MarkOwnerForDeletion(owner string, onlyIntended bool) *tree.LeafEntry { m.ctrl.T.Helper() @@ -949,3 +938,31 @@ func (mr *MockLeafVariantEntriesMockRecorder) MarkOwnerForDeletion(owner, onlyIn mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkOwnerForDeletion", reflect.TypeOf((*MockLeafVariantEntries)(nil).MarkOwnerForDeletion), owner, onlyIntended) } + +// RemoveDeletedByOwner mocks base method. +func (m *MockLeafVariantEntries) RemoveDeletedByOwner(owner string) *tree.LeafEntry { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveDeletedByOwner", owner) + ret0, _ := ret[0].(*tree.LeafEntry) + return ret0 +} + +// RemoveDeletedByOwner indicates an expected call of RemoveDeletedByOwner. +func (mr *MockLeafVariantEntriesMockRecorder) RemoveDeletedByOwner(owner any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDeletedByOwner", reflect.TypeOf((*MockLeafVariantEntries)(nil).RemoveDeletedByOwner), owner) +} + +// ResetFlags mocks base method. +func (m *MockLeafVariantEntries) ResetFlags(deleteFlag, newFlag, updatedFlag bool) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetFlags", deleteFlag, newFlag, updatedFlag) + ret0, _ := ret[0].(int) + return ret0 +} + +// ResetFlags indicates an expected call of ResetFlags. +func (mr *MockLeafVariantEntriesMockRecorder) ResetFlags(deleteFlag, newFlag, updatedFlag any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetFlags", reflect.TypeOf((*MockLeafVariantEntries)(nil).ResetFlags), deleteFlag, newFlag, updatedFlag) +} diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 1e030e09..f1efe758 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -93,7 +93,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca ) scb := schemaClient.NewSchemaClientBound(c.Schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { cancel() @@ -113,7 +113,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca deviationClients: make(map[sdcpb.DataServer_WatchDeviationsServer]string), syncTree: syncTreeRoot, syncTreeMutex: &sync.RWMutex{}, - taskPool: pool.NewSharedTaskPool(ctx, runtime.NumCPU()), + taskPool: pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)), } ds.transactionManager = types.NewTransactionManager(NewDatastoreRollbackAdapter(ds)) @@ -239,12 +239,12 @@ func (d *Datastore) BlameConfig(ctx context.Context, includeDefaults bool) (*sdc return nil, err } // load all intents - _, err = d.LoadAllButRunningIntents(ctx, root, true) + _, err = d.LoadAllButRunningIntents(ctx, root) if err != nil { return nil, err } - blamePool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + blamePool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) bcp := tree.NewBlameConfigProcessor(tree.NewBlameConfigProcessorConfig(includeDefaults)) bte, err := bcp.Run(ctx, root.GetRoot(), blamePool) diff --git a/pkg/datastore/deviations.go b/pkg/datastore/deviations.go index cf8e60bf..54f56867 100644 --- a/pkg/datastore/deviations.go +++ b/pkg/datastore/deviations.go @@ -123,12 +123,6 @@ func (d *Datastore) SendDeviations(ctx context.Context, ch <-chan *treetypes.Dev continue } - if log := log.V(logger.VTrace); log.Enabled() { - if deviation.Reason() == treetypes.DeviationReasonNotApplied { // TODO add check for trace level Trace - log.Info("NOT APPLIED", "path", deviation.Path().ToXPath(false), "actual value", deviation.CurrentValue().ToString(), "expected value", deviation.ExpectedValue().ToString(), "intent", deviation.IntentName()) - } - } - err := dc.Send(&sdcpb.WatchDeviationResponse{ Name: d.config.Name, Intent: deviation.IntentName(), @@ -169,7 +163,7 @@ func (d *Datastore) calculateDeviations(ctx context.Context) (<-chan *treetypes. return nil, err } - addedIntentNames, err := d.LoadAllButRunningIntents(ctx, deviationTree, true) + addedIntentNames, err := d.LoadAllButRunningIntents(ctx, deviationTree) if err != nil { return nil, err } diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 6ce84237..73cbed4e 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -66,7 +66,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } // otherwise consult cache - root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, intentName)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, d.taskPool)) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, tp.GetIntentName(), tp.GetPriority(), types.NewUpdateInsertFlags()) + _, err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index bb43a051..2c85cda2 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -2,6 +2,7 @@ package datastore import ( "context" + "sync" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" @@ -16,17 +17,44 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i log := logger.FromContext(ctx) d.syncTreeMutex.Lock() - defer d.syncTreeMutex.Unlock() + syncTreeUnlock := sync.OnceFunc(d.syncTreeMutex.Unlock) + + defer syncTreeUnlock() + + // create a virtual task pool for delete operations for _, delete := range deletes { - err := d.syncTree.DeleteBranch(ctx, delete, tree.RunningIntentName) + // navigate to delete path + deleteRoot, err := d.syncTree.NavigateSdcpbPath(ctx, delete) + if err != nil { + log.Error(err, "failed navigating to delete path", "path", delete.ToXPath(false)) + continue + } + // apply delete marker, setting owner delete flag on running intent + err = tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tree.RunningIntentName, false)).Run(deleteRoot, d.taskPool) if err != nil { - log.Error(err, "failed deleting path from datastore sync tree", "severity", "WARN", "path", delete.ToXPath(false)) + log.Error(err, "failed applying delete to path", "path", delete.ToXPath(false)) continue } } + // import new config if provided if importer != nil { - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, tree.RunningIntentName, tree.RunningValuesPrio, treetypes.NewUpdateInsertFlags()) + _, err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), d.taskPool) + if err != nil { + return err + } + } + + // run remove deleted processor to clean up entries marked as deleted by owner + delProcessorParams := tree.NewRemoveDeletedProcessorParameters(tree.RunningIntentName) + err := tree.NewRemoveDeletedProcessor(delProcessorParams).Run(d.syncTree.GetRoot(), d.taskPool) + if err != nil { + return err + } + + // delete entries that have zero-length leaf variant entries after remove deleted processing + for _, e := range delProcessorParams.GetZeroLengthLeafVariantEntries() { + err := e.GetParent().DeleteBranch(ctx, &sdcpb.Path{Elem: []*sdcpb.PathElem{sdcpb.NewPathElem(e.PathName(), nil)}}, tree.RunningIntentName) if err != nil { return err } @@ -34,7 +62,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i // conditional trace logging if log := log.V(logger.VTrace); log.Enabled() { - treeExport, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + treeExport, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err == nil { json, err := protojson.MarshalOptions{Multiline: false}.Marshal(treeExport) if err == nil { @@ -43,14 +71,82 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } } + // run reset flags processor to reset flags + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetDeleteFlag().SetNewFlag().SetUpdateFlag() + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(d.syncTree.GetRoot(), d.taskPool) + if err != nil { + return err + } + + // create a deep copy of the sync tree for revert operation + syncTreeCopy, err := d.syncTree.DeepCopy(ctx) + if err != nil { + return err + } + + // release the sync tree lock early, it is no longer needed + syncTreeUnlock() + + // perform the revert operation to apply changes to the device + // TODO: this should probably be executed in a separate goroutine + err = d.performRevert(ctx, syncTreeCopy) + if err != nil { + return err + } + return nil + } func (d *Datastore) NewEmptyTree(ctx context.Context) (*tree.RootEntry, error) { - tc := tree.NewTreeContext(d.schemaClient, tree.RunningIntentName) + tc := tree.NewTreeContext(d.schemaClient, d.taskPool) newTree, err := tree.NewTreeRoot(ctx, tc) if err != nil { return nil, err } return newTree, nil } + +func (d *Datastore) performRevert(ctx context.Context, t *tree.RootEntry) error { + log := logger.FromContext(ctx) + _, err := d.LoadAllButRunningIntents(ctx, t) + if err != nil { + return err + } + + err = t.FinishInsertionPhase(ctx) + if err != nil { + return err + } + + // TODO: optimize by checking only paths that where covered by the syncconfig + del, err := t.GetDeletes(true) + if err != nil { + return err + } + + // if we have deletes, we need to perform an apply + performApply := len(del) > 0 + + // if no deletes, check if we have updates + if !performApply { + updList, err := t.ToProtoUpdates(ctx, true) + if err != nil { + return err + } + // if the update list is non-empty, we need to perform an apply + performApply = len(updList) > 0 + } + + if performApply { + log.Info("reverting after sync") + resp, err := d.applyIntent(ctx, t) + if err != nil { + respJ := protojson.MarshalOptions{Multiline: false} + respStr, _ := respJ.Marshal(resp) + log.Error(err, "failed applying deviations to running", "response", string(respStr)) + } + } + + return nil +} diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go new file mode 100644 index 00000000..f6faf4d7 --- /dev/null +++ b/pkg/datastore/sync_test.go @@ -0,0 +1,428 @@ +package datastore + +import ( + "context" + "encoding/json" + "fmt" + "runtime" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/mocks/mockcacheclient" + "github.com/sdcio/data-server/mocks/mocktarget" + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/datastore/target" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree" + "github.com/sdcio/data-server/pkg/tree/importer" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" + "go.uber.org/mock/gomock" +) + +// TestApplyToRunning tests the ApplyToRunning method of the Datastore struct. +func TestApplyToRunning(t *testing.T) { + // Setup + ctx := context.Background() + + // Define test cases + tests := []struct { + name string + deletes []*sdcpb.Path + importerFunc func() importer.ImportConfigAdapter + syncTreeFunc func() *tree.RootEntry + resultFunc func() any + cacheClientFunc func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound + sbiFunc func(ctrl *gomock.Controller) target.Target + wantErr bool + }{ + { + name: "delete entire interface (e1-1 existed now e1-2 added)", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTreeFunc: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importerFunc: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/2": { + Name: ygot.String("ethernet-1/2"), + Description: ygot.String("1/2 description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) + }, + resultFunc: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/2": { + Name: ygot.String("ethernet-1/2"), + Description: ygot.String("1/2 description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, + }, + { + name: "delete description of existing interface", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTreeFunc: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importerFunc: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) + }, + resultFunc: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, + }, + { + name: "change description of existing interface", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTreeFunc: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importerFunc: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my other description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) + }, + resultFunc: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my other description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, + }, + } + + // Run tests + for _, tt := range tests { + + fmt.Println("----" + tt.name) + + t.Run(tt.name, func(t *testing.T) { + syncTree := tt.syncTreeFunc() + + ctrl := gomock.NewController(t) + + datastore := &Datastore{ + syncTreeMutex: &sync.RWMutex{}, + syncTree: syncTree, + taskPool: pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)), + cacheClient: tt.cacheClientFunc(ctrl), + sbi: tt.sbiFunc(ctrl), + } + + err := datastore.ApplyToRunning(ctx, tt.deletes, tt.importerFunc()) + if (err != nil) != tt.wantErr { + t.Errorf("ApplyToRunning() error = %v, wantErr %v", err, tt.wantErr) + } + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + + resultRoot, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := tt.resultFunc() + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + err = resultRoot.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + + fmt.Println(syncTree.String()) + + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetNewFlag().SetUpdateFlag() + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), datastore.taskPool) + if err != nil { + t.Fatalf("failed to reset flags: %v", err) + } + + fmt.Println("Adjusted flags count:", resetFlagsProcessorParams.GetAdjustedFlagsCount()) + + if diff := cmp.Diff(resultRoot.String(), syncTree.String()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/datastore/target/gnmi/get.go b/pkg/datastore/target/gnmi/get.go index d76bc0ad..ab5900e1 100644 --- a/pkg/datastore/target/gnmi/get.go +++ b/pkg/datastore/target/gnmi/get.go @@ -142,7 +142,7 @@ func (s *GetSync) internalGetSync(req *sdcpb.GetDataRequest) { return } - result, err := s.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + result, err := s.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err != nil { log.Error(err, "failure exporting synctree") return diff --git a/pkg/datastore/target/gnmi/stream.go b/pkg/datastore/target/gnmi/stream.go index 03da073a..546e2119 100644 --- a/pkg/datastore/target/gnmi/stream.go +++ b/pkg/datastore/target/gnmi/stream.go @@ -148,7 +148,7 @@ func (s *StreamSync) buildTreeSyncWithDatastore(cUS <-chan *NotificationData, sy if err != nil { log.Error(err, "failed adding update to synctree") } - syncTree.AddExplicitDeletes(tree.RunningIntentName, tree.RunningValuesPrio, noti.deletes) + syncTree.GetTreeContext().AddExplicitDeletes(tree.RunningIntentName, tree.RunningValuesPrio, noti.deletes) case <-syncResponse: syncTree, err = s.syncToRunning(syncTree, syncTreeMutex, true) tickerActive = true @@ -176,7 +176,7 @@ func (s *StreamSync) gnmiSubscribe(subReq *gnmi.SubscribeRequest, updChan chan<- respChan, errChan := s.target.Subscribe(s.ctx, subReq, s.config.Name) - taskPool := s.vpoolFactory.NewVirtualPool(pool.VirtualTolerant, 10) + taskPool := s.vpoolFactory.NewVirtualPool(pool.VirtualTolerant) defer taskPool.CloseForSubmit() taskParams := NewNotificationProcessorTaskParameters(updChan, s.schemaClient) @@ -221,7 +221,7 @@ func (s *StreamSync) syncToRunning(syncTree *tree.RootEntry, m *sync.Mutex, logC defer m.Unlock() startTime := time.Now() - result, err := syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + result, err := syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) log.V(logger.VTrace).Info("exported tree", "tree", result.String()) if err != nil { diff --git a/pkg/datastore/target/netconf/nc.go b/pkg/datastore/target/netconf/nc.go index 1f77aa0f..fea9b7ab 100644 --- a/pkg/datastore/target/netconf/nc.go +++ b/pkg/datastore/target/netconf/nc.go @@ -30,6 +30,7 @@ import ( "github.com/sdcio/data-server/pkg/datastore/target/netconf/driver/scrapligo" nctypes "github.com/sdcio/data-server/pkg/datastore/target/netconf/types" "github.com/sdcio/data-server/pkg/datastore/target/types" + "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/importer/xml" ) @@ -88,9 +89,9 @@ func (t *ncTarget) GetImportAdapter(ctx context.Context, req *sdcpb.GetDataReque return nil, err } - cmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root()) + xmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root(), tree.RunningIntentName, tree.RunningValuesPrio, false) - return cmlImport, nil + return xmlImport, nil } func (t *ncTarget) internalGet(ctx context.Context, req *sdcpb.GetDataRequest) (*nctypes.NetconfResponse, error) { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 4cb07e20..9e5c7fd7 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -9,7 +9,6 @@ import ( "time" "github.com/sdcio/data-server/pkg/datastore/types" - "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" treeproto "github.com/sdcio/data-server/pkg/tree/importer/proto" treetypes "github.com/sdcio/data-server/pkg/tree/types" @@ -41,12 +40,15 @@ func (d *Datastore) SdcpbTransactionIntentToInternalTI(ctx context.Context, req if req.GetOrphan() { ti.SetDeleteOnlyIntendedFlag() } - if req.GetDeviation() { - ti.SetDeviation() + if req.GetNonRevertive() { + ti.SetNonRevertive() } if req.GetDeleteIgnoreNoExist() { ti.SetDeleteIgnoreNonExisting() } + if req.GetPreviouslyApplied() { + ti.SetPreviouslyApplied() + } // convert the sdcpb.updates to tree.UpdateSlice Updates, err := treetypes.ExpandAndConvertIntent(ctx, d.schemaClient, req.GetIntent(), req.GetPriority(), req.GetUpdate(), time.Now().Unix()) @@ -70,7 +72,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa ctx = logf.IntoContext(ctx, log) // create a new TreeContext - tc := tree.NewTreeContext(d.schemaClient, d.Name()) + tc := tree.NewTreeContext(d.schemaClient, d.taskPool) // create a new TreeRoot to collect validate and hand to SBI.Set() root, err := tree.NewTreeRoot(ctx, tc) @@ -78,15 +80,12 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa return nil, err } - // set TreeContext actual owner to the const of ReplaceIntentName - tc.SetActualOwner(tree.ReplaceIntentName) - // store the actual / old running in the transaction runningProto, err := d.cacheClient.IntentGet(ctx, tree.RunningIntentName) if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), tree.RunningIntentName, tree.RunningValuesPrio, treetypes.NewUpdateInsertFlags()) + _, err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -147,7 +146,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa return warnings, nil } -func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry, excludeDeviations bool) ([]string, error) { +func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry) ([]string, error) { log := logf.FromContext(ctx) intentNames := []string{} @@ -174,15 +173,12 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo IntentChan = nil break selectLoop } - if excludeDeviations && intent.Deviation { - continue - } log.V(logf.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) log.V(logf.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.FormatProtoJSON(intent)) intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, intent.GetIntentName(), intent.GetPriority(), treetypes.NewUpdateInsertFlags()) + _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -204,7 +200,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ return nil, err } - _, err = d.LoadAllButRunningIntents(ctx, root, false) + _, err = d.LoadAllButRunningIntents(ctx, root) if err != nil { return nil, err } @@ -213,6 +209,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ flagNew := treetypes.NewUpdateInsertFlags() // where the New flag is set flagNew.SetNewFlag() + flagExisting := treetypes.NewUpdateInsertFlags() // iterate through all the intents for _, intent := range transaction.GetNewIntents() { @@ -222,16 +219,15 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ oldIntentContent := lvs.ToPathAndUpdateSlice() - deleteVisitorPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(intent.GetName(), intent.GetOnlyIntended())) - err := ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err := ownerDeleteMarker.Run(root.GetRoot(), d.taskPool) if err != nil { return nil, err } // clear the owners existing explicit delete entries, retrieving the old entries for storing in the transaction for possible rollback - oldExplicitDeletes := root.RemoveExplicitDeletes(intent.GetName()) + oldExplicitDeletes := root.GetTreeContext().RemoveExplicitDeletes(intent.GetName()) priority := int32(math.MaxInt32) if len(oldIntentContent) > 0 { @@ -245,15 +241,23 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ } if !intent.GetDeleteFlag() { + flag := flagNew + // determine the correct flag to use based on whether the intent is non-revertive and was previously applied + if intent.NonRevertive() && intent.GetPreviouslyApplied() { + flag = flagExisting + } + // add the content to the Tree - err = root.AddUpdatesRecursive(ctx, intent.GetUpdates(), flagNew) + err = root.AddUpdatesRecursive(ctx, intent.GetUpdates(), flag) if err != nil { return nil, err } // add the explicit delete entries - root.AddExplicitDeletes(intent.GetName(), intent.GetPriority(), intent.GetDeletes()) + root.GetTreeContext().AddExplicitDeletes(intent.GetName(), intent.GetPriority(), intent.GetDeletes()) } + + root.SetNonRevertiveIntent(intent.GetName(), intent.NonRevertive()) } les := tree.LeafVariantSlice{} @@ -354,7 +358,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ delSl := deletesOwner.ToXPathSlice() log.V(logf.VTrace).Info("deletes owner", "deletes-owner", delSl) - protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority(), intent.Deviation()) + protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority()) switch { case errors.Is(err, tree.ErrorIntentNotPresent): err = d.cacheClient.IntentDelete(ctx, intent.GetName(), intent.GetDeleteIgnoreNonExisting()) @@ -420,7 +424,7 @@ func (d *Datastore) writeBackSyncTree(ctx context.Context, updates tree.LeafVari } // export the synctree - newRunningIntent, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + newRunningIntent, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err != nil && err != tree.ErrorIntentNotPresent { return err } diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go new file mode 100644 index 00000000..2dc0ee12 --- /dev/null +++ b/pkg/datastore/transaction_rpc_test.go @@ -0,0 +1,226 @@ +package datastore + +import ( + "context" + "encoding/json" + "runtime" + "sync" + "testing" + "time" + + "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/mocks/mockcacheclient" + "github.com/sdcio/data-server/mocks/mocktarget" + "github.com/sdcio/data-server/pkg/config" + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/datastore/types" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + treetypes "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" + "go.uber.org/mock/gomock" +) + +func TestTransactionSet_PreviouslyApplied(t *testing.T) { + ctx := context.Background() + + // Setup Schema + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + + // Setup Running Config Data + runningDevice := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + runningJson, err := ygot.EmitJSON(runningDevice, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal running config: %v", err) + } + var runningAny any + json.Unmarshal([]byte(runningJson), &runningAny) + + // Setup Intent Data (Same as Running) + intentStrSame := runningJson // Same content + + // Setup Intent Data (Different) + deviceDiff := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("new description"), + }, + }, + } + intentStrDiff, _ := ygot.EmitJSON(deviceDiff, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + + tests := []struct { + name string + previouslyApplied bool + nonRevertive bool + intentStr string + expectUpdates bool + }{ + { + name: "Revertive - Not Previously Applied - Should Produce Updates (Redundant)", + previouslyApplied: false, + nonRevertive: false, + intentStr: intentStrSame, + expectUpdates: true, + }, + { + name: "Revertive - Previously Applied - Should Produce Updates (PA Ignored)", + previouslyApplied: true, + nonRevertive: false, + intentStr: intentStrSame, + expectUpdates: true, + }, + { + name: "Revertive - Previously Applied - But Value Changed - Should Produce Updates", + previouslyApplied: true, + nonRevertive: false, + intentStr: intentStrDiff, + expectUpdates: true, + }, + { + name: "NonRevertive - Previously Applied - Value Changed - Should Produce NO Updates", + previouslyApplied: true, + nonRevertive: true, + intentStr: intentStrDiff, + expectUpdates: false, + }, + { + name: "NonRevertive - Not Previously Applied - Value Changed - Should Produce Updates", + previouslyApplied: false, + nonRevertive: true, + intentStr: intentStrDiff, + expectUpdates: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Setup Mock Cache Client + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + // Expect IntentGetAll (called by LoadAllButRunningIntents) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + + // Expect IntentModify (called by TransactionSet to save intent) + ccb.EXPECT().IntentModify(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + // Setup Mock SBI + sbi := mocktarget.NewMockTarget(ctrl) + // Expect Set if updates are expected or if dryRun is false (we will use dryRun=false) + // Actually TransactionSet calls applyIntent which calls sbi.Set + sbi.EXPECT().Set(gomock.Any(), gomock.Any()).Return(&sdcpb.SetDataResponse{}, nil).AnyTimes() + + // Setup SyncTree with Running Config + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create sync tree root: %v", err) + } + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + // Populate SyncTree with Running config + _, err = syncTreeRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(runningAny, tree.RunningIntentName, tree.RunningValuesPrio, false), treetypes.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import running config: %v", err) + } + err = syncTreeRoot.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + // Reset flags on syncTree so everything is "existing" + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetNewFlag().SetUpdateFlag() + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTreeRoot.GetRoot(), vpf) + if err != nil { + t.Fatalf("failed to reset flags: %v", err) + } + + // Setup Datastore + ds := &Datastore{ + config: &config.DatastoreConfig{ + Validation: config.NewValidationConfig(), + Name: "test-ds", + }, + syncTreeMutex: &sync.RWMutex{}, + syncTree: syncTreeRoot, // Pre-populated syncTree + taskPool: vpf, + cacheClient: ccb, + sbi: sbi, + dmutex: &sync.Mutex{}, + schemaClient: scb, + } + ds.transactionManager = types.NewTransactionManager(NewDatastoreRollbackAdapter(ds)) + + // Prepare Transaction Request + transactionId := "txn-1" + + // Build TransactionIntent from req using helper or manually + // We need types.TransactionIntent + intentName := "intent1" + priority := int32(10) + + ti := types.NewTransactionIntent(intentName, priority) + if tt.previouslyApplied { + ti.SetPreviouslyApplied() + } + if tt.nonRevertive { + ti.SetNonRevertive() + } + + // Parse updates from intentStr + updates, err := treetypes.ExpandAndConvertIntent(ctx, scb, intentName, priority, []*sdcpb.Update{{ + Path: &sdcpb.Path{}, + Value: &sdcpb.TypedValue{ + Value: &sdcpb.TypedValue_JsonVal{ + JsonVal: []byte(tt.intentStr), + }, + }, + }}, time.Now().Unix()) + if err != nil { + t.Fatalf("failed to expand intent: %v", err) + } + ti.AddUpdates(updates) + + transactionIntents := []*types.TransactionIntent{ti} + + // Call TransactionSet + resp, err := ds.TransactionSet(ctx, transactionId, transactionIntents, nil, 10*time.Second, false) + if err != nil { + t.Fatalf("TransactionSet failed: %v", err) + } + + // Verify Updates + hasUpdates := len(resp.GetUpdate()) > 0 + if hasUpdates != tt.expectUpdates { + t.Errorf("Expected updates: %v, got: %v (count: %d)\nUpdates: %v", tt.expectUpdates, hasUpdates, len(resp.GetUpdate()), resp.GetUpdate()) + } + }) + } +} diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index b9ea06e2..a3e3be45 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -75,6 +75,7 @@ func TestDatastore_populateTree(t *testing.T) { intendedStoreUpdates []*types.PathAndUpdate runningStoreUpdates []*types.PathAndUpdate NotOnlyNewOrUpdated bool // it negated when used in the call, usually we want it to be true + nonRevertive bool }{ { name: "DoubleKey - Delete Single item", @@ -1470,7 +1471,7 @@ func TestDatastore_populateTree(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tt.intentName) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -1502,11 +1503,9 @@ func TestDatastore_populateTree(t *testing.T) { t.Error(err) } - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tt.intentName, false)) - - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1514,7 +1513,10 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny), tt.intentName, tt.intentPrio, newFlag) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + + _, err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, sharedPool) + if err != nil { t.Error(err) } @@ -1526,7 +1528,6 @@ func TestDatastore_populateTree(t *testing.T) { } fmt.Println(root.String()) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) fmt.Printf("Validation Errors:\n%v\n", strings.Join(validationResult.ErrorsStr(), "\n")) diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index ed93d911..88acd4ef 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -59,6 +59,7 @@ func TestDatastore_validateTree(t *testing.T) { intendedStoreUpdates []*cache.Update NotOnlyNewOrUpdated bool // it negated when used in the call, usually we want it to be true expectedWarnings []string + nonRevertive bool }{ { @@ -186,7 +187,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - tc := tree.NewTreeContext(scb, tt.intentName) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -195,9 +196,10 @@ func TestDatastore_validateTree(t *testing.T) { flagsNew := types.NewUpdateInsertFlags() flagsNew.SetNewFlag() - importer := json_importer.NewJsonTreeImporter(jsonConf) + importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) - err = root.ImportConfig(ctx, path, importer, tt.intentName, tt.intentPrio, flagsNew) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, path, importer, flagsNew, vpf) if err != nil { t.Error(err) } @@ -207,7 +209,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) t.Log(root.String()) diff --git a/pkg/datastore/types/transaction_intent.go b/pkg/datastore/types/transaction_intent.go index 31a77ba9..674a7ec1 100644 --- a/pkg/datastore/types/transaction_intent.go +++ b/pkg/datastore/types/transaction_intent.go @@ -11,13 +11,12 @@ type TransactionIntent struct { updates []*treetypes.PathAndUpdate delete bool // onlyIntended, the orphan flag, delte only from intended store, but keep in device - onlyIntended bool - priority int32 - // deviation indicates that the intent is a tolerated deviation. - // it will be stored and used for change calculation but will be excluded when claculating actual deviations. - deviation bool + onlyIntended bool + priority int32 + nonRevertive bool deleteIgnoreNonExisting bool explicitDeletes *sdcpb.PathSet + previouslyApplied bool } func NewTransactionIntent(name string, priority int32) *TransactionIntent { @@ -53,16 +52,20 @@ func (ti *TransactionIntent) GetDeletes() *sdcpb.PathSet { return ti.explicitDeletes } +func (ti *TransactionIntent) GetPreviouslyApplied() bool { + return ti.previouslyApplied +} + func (ti *TransactionIntent) GetOnlyIntended() bool { return ti.onlyIntended } -func (ti *TransactionIntent) SetDeviation() { - ti.deviation = true +func (ti *TransactionIntent) SetNonRevertive() { + ti.nonRevertive = true } -func (ti *TransactionIntent) Deviation() bool { - return ti.deviation +func (ti *TransactionIntent) NonRevertive() bool { + return ti.nonRevertive } func (ti *TransactionIntent) SetDeleteIgnoreNonExisting() { @@ -77,6 +80,10 @@ func (ti *TransactionIntent) SetDeleteFlag() { ti.delete = true } +func (ti *TransactionIntent) SetPreviouslyApplied() { + ti.previouslyApplied = true +} + func (ti *TransactionIntent) SetDeleteOnlyIntendedFlag() { ti.delete = true ti.onlyIntended = true diff --git a/pkg/dslog/logger.go b/pkg/dslog/logger.go deleted file mode 100644 index c5adb9fc..00000000 --- a/pkg/dslog/logger.go +++ /dev/null @@ -1,5 +0,0 @@ -package dslog - -const ( - TraceLevel = -8 -) diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index 18f224eb..7b762dd1 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -34,10 +34,10 @@ type Pool[T any] struct { inflightC *sync.Cond } -// NewWorkerPool creates a new Pool. If workerCount <= 0 it defaults to runtime.NumCPU(). +// NewWorkerPool creates a new Pool. If workerCount <= 0 it defaults to runtime.GOMAXPROCS(0). func NewWorkerPool[T any](parent context.Context, workerCount int) *Pool[T] { if workerCount <= 0 { - workerCount = runtime.NumCPU() + workerCount = runtime.GOMAXPROCS(0) } ctx, cancel := context.WithCancel(parent) p := &Pool[T]{ diff --git a/pkg/pool/virtual_pool.go b/pkg/pool/virtual_pool.go index 850f4124..a3f0fdf7 100644 --- a/pkg/pool/virtual_pool.go +++ b/pkg/pool/virtual_pool.go @@ -42,20 +42,14 @@ func (f TaskFunc) Run(ctx context.Context, submit func(Task) error) error { // --- ErrorCollector (per-virtual tolerant mode) --- // ErrorCollector collects errors for a virtual pool. -// It stores a snapshotable slice and provides a live channel for streaming. +// It stores a snapshotable slice of errors. type ErrorCollector struct { mu sync.Mutex errs []error - Ch chan error } -func newErrorCollector(buf int) *ErrorCollector { - if buf <= 0 { - buf = 1024 - } - return &ErrorCollector{ - Ch: make(chan error, buf), - } +func newErrorCollector() *ErrorCollector { + return &ErrorCollector{} } func (ec *ErrorCollector) add(err error) { @@ -65,12 +59,6 @@ func (ec *ErrorCollector) add(err error) { ec.mu.Lock() defer ec.mu.Unlock() ec.errs = append(ec.errs, err) - - select { - case ec.Ch <- err: - default: - // drop if full - } } // Errors returns a snapshot of collected errors. @@ -82,11 +70,6 @@ func (ec *ErrorCollector) Errors() []error { return out } -// close channel when done -func (ec *ErrorCollector) close() { - close(ec.Ch) -} - // --- Virtual pool system --- var ErrVirtualPoolClosed = errors.New("virtual pool closed for submit") @@ -135,13 +118,8 @@ func (s *SharedTaskPool) Wait() error { } // NewVirtualPool creates and registers a virtual pool on top of the shared pool. -// id is an arbitrary identifier (must be unique per SharedTaskPool). -// mode controls failure semantics. buf controls error channel buffer for tolerant mode. -func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI { - // ensure unique id in the shared pool's map. If the requested id is already - // registered, append a short random hex postfix so multiple callers can - // create virtual pools with the same base name without colliding. - +// mode controls failure semantics. +func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode) VirtualPoolI { v := &VirtualPool{ parent: s, mode: mode, @@ -151,7 +129,7 @@ func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI done: make(chan struct{}), } if mode == VirtualTolerant { - v.ec = newErrorCollector(buf) + v.ec = newErrorCollector() } return v } @@ -175,9 +153,7 @@ type VirtualPool struct { // firstErr used for fail-fast firstErr atomic.Pointer[error] // per-virtual inflight counter (matches lifecycle of tasks submitted by this virtual) - inflight int64 - // ensure collector channel closed only once - collectorOnce sync.Once + inflight atomic.Int64 // ensure done channel closed only once (for Wait) waitOnce sync.Once // done is closed when the virtual pool is closed for submit and inflight reaches zero @@ -238,7 +214,7 @@ func (vt *virtualTask) Run(ctx context.Context, submit func(Task) error) error { func (v *VirtualPool) Submit(t Task) error { // Increment inflight BEFORE checking closed to avoid race where CloseForSubmit // sees inflight=0 and closes the pool while we are in the middle of submitting. - atomic.AddInt64(&v.inflight, 1) + v.inflight.Add(1) // fast-fail if virtual pool closed for submit if v.closed.Load() { @@ -261,12 +237,7 @@ func (v *VirtualPool) Submit(t Task) error { } func (v *VirtualPool) decrementInflight() { - if remaining := atomic.AddInt64(&v.inflight, -1); remaining == 0 && v.closed.Load() { - v.collectorOnce.Do(func() { - if v.ec != nil { - v.ec.close() - } - }) + if remaining := v.inflight.Add(-1); remaining == 0 && v.closed.Load() { v.waitOnce.Do(func() { close(v.done) }) @@ -287,7 +258,7 @@ func (v *VirtualPool) submitInternal(t Task) error { return ErrVirtualPoolClosed } // increment per-virtual inflight (will be decremented by worker after run) - atomic.AddInt64(&v.inflight, 1) + v.inflight.Add(1) vt := &virtualTask{vp: v, task: t} if err := v.parent.submitWrapped(vt); err != nil { // submission failed: revert inflight @@ -302,14 +273,8 @@ func (v *VirtualPool) submitInternal(t Task) error { // when all virtual pools are done (call SharedTaskPool.CloseForSubmit()). func (v *VirtualPool) CloseForSubmit() { v.closed.Store(true) - // if nothing inflight, close collector now - if atomic.LoadInt64(&v.inflight) == 0 { - v.collectorOnce.Do(func() { - if v.ec != nil { - v.ec.close() - } - }) - // signal Wait() callers that virtual is drained + // if nothing inflight, signal Wait() callers that virtual is drained + if v.inflight.Load() == 0 { v.waitOnce.Do(func() { close(v.done) }) @@ -323,6 +288,13 @@ func (v *VirtualPool) Wait() { <-v.done } +// CloseAndWait is a convenience method that closes the virtual pool for submission +// and waits for all inflight tasks to complete. +func (v *VirtualPool) CloseAndWait() { + v.CloseForSubmit() + v.Wait() +} + // isFailed returns true if this virtual pool encountered a fail-fast error. func (v *VirtualPool) isFailed() bool { if p := v.firstErr.Load(); p != nil && *p != nil { @@ -346,18 +318,13 @@ func (v *VirtualPool) FirstError() error { } // Errors returns a snapshot of collected errors for tolerant virtual pools. -// For fail-fast virtual pools this returns nil. +// For fail-fast virtual pools this returns the first error if any. func (v *VirtualPool) Errors() []error { if v.ec == nil { + if err := v.FirstError(); err != nil { + return []error{err} + } return nil } return v.ec.Errors() } - -// ErrorChan returns the live channel of errors for tolerant mode, or nil for fail-fast mode. -func (v *VirtualPool) ErrorChan() <-chan error { - if v.ec == nil { - return nil - } - return v.ec.Ch -} diff --git a/pkg/pool/virtual_pool_iface.go b/pkg/pool/virtual_pool_iface.go index 396a34c2..1b2f9bb2 100644 --- a/pkg/pool/virtual_pool_iface.go +++ b/pkg/pool/virtual_pool_iface.go @@ -12,17 +12,17 @@ type VirtualPoolI interface { CloseForSubmit() // Wait blocks until the virtual has been closed for submit and all inflight tasks have completed. Wait() + // CloseAndWait is a convenience method that closes for submission and waits for drain. + CloseAndWait() // FirstError returns the first encountered error for fail-fast virtual pools, or nil. FirstError() error // Errors returns a snapshot of collected errors for tolerant virtual pools. Errors() []error - // ErrorChan returns the live channel of errors for tolerant mode, or nil for fail-fast mode. - ErrorChan() <-chan error } // Ensure VirtualPool implements the interface. var _ VirtualPoolI = (*VirtualPool)(nil) type VirtualPoolFactory interface { - NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI + NewVirtualPool(mode VirtualMode) VirtualPoolI } diff --git a/pkg/pool/virtual_pool_test.go b/pkg/pool/virtual_pool_test.go index 086abc2c..acedca3a 100644 --- a/pkg/pool/virtual_pool_test.go +++ b/pkg/pool/virtual_pool_test.go @@ -51,8 +51,8 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { sp := NewSharedTaskPool(ctx, 4) // create virtual pools - vt := sp.NewVirtualPool(VirtualTolerant, 16) - vf := sp.NewVirtualPool(VirtualFailFast, 0) + vt := sp.NewVirtualPool(VirtualTolerant) + vf := sp.NewVirtualPool(VirtualFailFast) // submit tasks: tolerant pool will collect errors, fail pool will stop after first error var cntT int64 @@ -88,30 +88,10 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { // close shared pool for submit and wait sp.CloseForSubmit() - // drain tolerant error channel live - var seenErrs int32 - done := make(chan struct{}) - go func() { - for e := range vt.ErrorChan() { - if e != nil { - atomic.AddInt32(&seenErrs, 1) - } - } - close(done) - }() - if err := sp.Wait(); err != nil { t.Fatalf("shared wait error: %v", err) } - // tolerant collector channel is closed automatically when virtual is closed and inflight reaches zero - // wait for drain goroutine - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatalf("timeout waiting for tolerant drain") - } - // tolerant: expect 4 increments (one of the tasks had nested:1 which is now allowed even after CloseForSubmit) if got := atomic.LoadInt64(&cntT); got != 4 { t.Fatalf("tolerant counter expected 4 got %d", got) @@ -135,7 +115,7 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { func TestVirtualPool_Wait_Tolerant(t *testing.T) { ctx := context.Background() sp := NewSharedTaskPool(ctx, 2) - vt := sp.NewVirtualPool(VirtualTolerant, 4) + vt := sp.NewVirtualPool(VirtualTolerant) var cnt int64 // submit a few tasks @@ -157,16 +137,6 @@ func TestVirtualPool_Wait_Tolerant(t *testing.T) { // Wait on virtual specifically vt.Wait() - // ensure error channel closed (collector closed) - select { - case _, ok := <-vt.ErrorChan(): - if ok { - t.Fatalf("expected error channel to be closed") - } - default: - // channel may be drained; ensure snapshot is usable - } - if got := atomic.LoadInt64(&cnt); got != 2 { t.Fatalf("expected 2 increments, got %d", got) } @@ -176,7 +146,7 @@ func TestVirtualPool_Wait_Tolerant(t *testing.T) { func TestVirtualPool_Wait_FailFast(t *testing.T) { ctx := context.Background() sp := NewSharedTaskPool(ctx, 2) - vf := sp.NewVirtualPool(VirtualFailFast, 0) + vf := sp.NewVirtualPool(VirtualFailFast) var cnt int64 // submit a task that will fail and another that would be skipped when run @@ -207,7 +177,7 @@ func TestVirtualPool_Wait_FailFast(t *testing.T) { func TestVirtualPool_CancellationHang(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) sp := NewSharedTaskPool(ctx, 1) - vp := sp.NewVirtualPool(VirtualFailFast, 0) + vp := sp.NewVirtualPool(VirtualFailFast) var wg sync.WaitGroup wg.Add(2) diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index dd6fef22..6dedbe03 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -6,11 +6,9 @@ import ( "github.com/beevik/etree" "github.com/sdcio/data-server/pkg/config" - "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" - "github.com/sdcio/sdc-protos/tree_persist" - sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" ) const ( @@ -23,8 +21,8 @@ const ( ReplaceIntentName = "replace" ) -// newEntry constructor for Entries -func newEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeContext) (*sharedEntryAttributes, error) { +// NewEntry constructor for Entries +func NewEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeContext) (*sharedEntryAttributes, error) { // create a new sharedEntryAttributes instance sea, err := newSharedEntryAttributes(ctx, parent, pathElemName, tc) if err != nil { @@ -32,7 +30,7 @@ func newEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeCo } // add the Entry as a child to the parent Entry - err = parent.addChild(ctx, sea) + err = parent.AddChild(ctx, sea) return sea, err } @@ -43,7 +41,7 @@ type Entry interface { // GetLevel returns the depth of the Entry in the tree GetLevel() int // addChild Add a child entry - addChild(context.Context, Entry) error + AddChild(context.Context, Entry) error // getOrCreateChilds retrieves the sub-child pointed at by the path. // if the path does not exist in its full extend, the entries will be added along the way // if the path does not point to a schema defined path an error will be raise @@ -66,8 +64,6 @@ type Entry interface { // MarkOwnerDelete(o string, onlyIntended bool) // GetDeletes returns the cache-updates that are not updated, have no lower priority value left and hence should be deleted completely GetDeletes(entries []types.DeleteEntry, aggregatePaths bool) ([]types.DeleteEntry, error) - // Walk takes the EntryVisitor and applies it to every Entry in the tree - Walk(ctx context.Context, v EntryVisitor) error // Validate kicks off validation ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) // validateMandatory the Mandatory schema field @@ -106,7 +102,7 @@ type Entry interface { // as part of the SetIntent process. We need to consider this, when evaluating e.g. LeafRefs. // The returned boolean will in indicate if the value remains existing (true) after the setintent. // Or will disappear from device (running) as part of the update action. - remainsToExist() bool + RemainsToExist() bool // shouldDelete returns true if an explicit delete should be issued for the given branch shouldDelete() bool // canDelete checks if the entry can be Deleted. @@ -115,7 +111,7 @@ type Entry interface { // - remainsToExists() returns true, because they remain to exist even though implicitly. // - shouldDelete() returns false, because no explicit delete should be issued for them. canDelete() bool - GetChilds(DescendMethod) EntryMap + GetChilds(types.DescendMethod) EntryMap GetChild(name string) (Entry, bool) // entry, exists FilterChilds(keys map[string]string) ([]Entry, error) // ToJson returns the Tree contained structure as JSON @@ -130,8 +126,6 @@ type Entry interface { // ToXML returns the tree and its current state in the XML representation used by netconf ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) - // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - ImportConfig(ctx context.Context, importer importer.ImportConfigAdapterElement, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) @@ -145,14 +139,9 @@ type Entry interface { // returns true if the Entry contains leafvariants (presence container, field or leaflist) HoldsLeafvariants() bool - canDeleteBranch(keepDefault bool) bool - deleteCanDeleteChilds(keepDefault bool) -} - -type EntryVisitor interface { - DescendMethod() DescendMethod - Visit(ctx context.Context, e Entry) error - Up() + CanDeleteBranch(keepDefault bool) bool + DeleteCanDeleteChilds(keepDefault bool) + GetTreeContext() *TreeContext } type LeafVariantEntry interface { @@ -163,12 +152,16 @@ type LeafVariantEntry interface { type LeafVariantEntries interface { MarkOwnerForDeletion(owner string, onlyIntended bool) *LeafEntry + ResetFlags(deleteFlag bool, newFlag bool, updatedFlag bool) int GetHighestPrecedence(onlyNewOrUpdated bool, includeDefaults bool, includeExplicitDeletes bool) *LeafEntry GetRunning() *LeafEntry DeleteByOwner(owner string) *LeafEntry AddExplicitDeleteEntry(owner string, priority int32) *LeafEntry GetByOwner(owner string) *LeafEntry + RemoveDeletedByOwner(owner string) *LeafEntry Add(l *LeafEntry) + AddWithStats(l *LeafEntry, stats *types.ImportStats) + Length() int } type DescendMethod int diff --git a/pkg/tree/entry_map.go b/pkg/tree/entry_map.go index ecdccad8..59fa6832 100644 --- a/pkg/tree/entry_map.go +++ b/pkg/tree/entry_map.go @@ -1,3 +1,16 @@ package tree +import ( + "sort" +) + type EntryMap map[string]Entry + +func (e EntryMap) SortedKeys() []string { + keys := make([]string, 0, len(e)) + for k := range e { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index 70dc0185..c497b2d6 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -66,7 +66,7 @@ func Test_Entry(t *testing.T) { u2 := types.NewUpdate(nil, desc, int32(99), "me", int64(444)) u3 := types.NewUpdate(nil, desc, int32(98), "me", int64(88)) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -174,7 +174,7 @@ func Test_Entry_One(t *testing.T) { u3 := types.NewUpdate(nil, desc3, prio50, owner2, ts1) u3_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner2, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -300,7 +300,7 @@ func Test_Entry_Two(t *testing.T) { u1 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) u1_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -495,7 +495,7 @@ func Test_Entry_Three(t *testing.T) { u3r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) u4r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -566,11 +566,10 @@ func Test_Entry_Three(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -785,7 +784,7 @@ func Test_Entry_Four(t *testing.T) { } // u2o2_1 := types.NewUpdate(p2o2_1, testhelper.GetUIntTvProto(11), prio55, owner2, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -841,11 +840,10 @@ func Test_Entry_Four(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -958,7 +956,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -997,7 +995,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Log(root.String()) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1011,7 +1009,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1046,7 +1044,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { } } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1060,7 +1058,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Four", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1101,7 +1099,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { } } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1187,7 +1185,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } u6 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1216,11 +1214,10 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } } - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1483,7 +1480,7 @@ func Test_Schema_Population(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1538,7 +1535,7 @@ func Test_sharedEntryAttributes_SdcpbPath(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1669,7 +1666,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1692,7 +1689,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1708,7 +1705,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1730,7 +1727,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1805,7 +1802,7 @@ func Test_Validation_Deref(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1827,7 +1824,7 @@ func Test_Validation_Deref(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1861,7 +1858,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Valid", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1926,7 +1923,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // Should have no errors - all keys match their respective patterns @@ -1939,7 +1936,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_Owner", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1983,7 +1980,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_NextHop", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/importer/import_config_adapter.go b/pkg/tree/importer/import_config_adapter.go index a98f4abb..97418345 100644 --- a/pkg/tree/importer/import_config_adapter.go +++ b/pkg/tree/importer/import_config_adapter.go @@ -11,6 +11,9 @@ import ( type ImportConfigAdapter interface { ImportConfigAdapterElement GetDeletes() *sdcpb.PathSet + GetName() string + GetPriority() int32 + GetNonRevertive() bool } type ImportConfigAdapterElement interface { diff --git a/pkg/tree/importer/json/json_tree_importer.go b/pkg/tree/importer/json/json_tree_importer.go index 37e9073b..c002251b 100644 --- a/pkg/tree/importer/json/json_tree_importer.go +++ b/pkg/tree/importer/json/json_tree_importer.go @@ -12,29 +12,50 @@ import ( ) type JsonTreeImporter struct { - data any - name string + *JsonTreeImporterElement + intentName string + priority int32 + nonRevertive bool +} + +func (j *JsonTreeImporter) GetPriority() int32 { + return j.priority +} + +func (j *JsonTreeImporter) GetNonRevertive() bool { + return j.nonRevertive +} + +func (j *JsonTreeImporter) GetName() string { + return j.intentName } -func newJsonTreeImporterInternal(name string, d any) *JsonTreeImporter { +func NewJsonTreeImporter(d any, intentName string, priority int32, nonRevertive bool) *JsonTreeImporter { return &JsonTreeImporter{ - data: d, - name: name, + JsonTreeImporterElement: newJsonTreeImporterElement("root", d), + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, } } -func NewJsonTreeImporter(d any) *JsonTreeImporter { - return &JsonTreeImporter{ +type JsonTreeImporterElement struct { + data any + name string +} + +func newJsonTreeImporterElement(name string, d any) *JsonTreeImporterElement { + return &JsonTreeImporterElement{ data: d, - name: "root", + name: name, } } -func (j *JsonTreeImporter) GetDeletes() *sdcpb.PathSet { +func (j *JsonTreeImporterElement) GetDeletes() *sdcpb.PathSet { return sdcpb.NewPathSet() } -func (j *JsonTreeImporter) GetElement(key string) importer.ImportConfigAdapterElement { +func (j *JsonTreeImporterElement) GetElement(key string) importer.ImportConfigAdapterElement { switch d := j.data.(type) { case map[string]any: @@ -44,14 +65,14 @@ func (j *JsonTreeImporter) GetElement(key string) importer.ImportConfigAdapterEl elemName = beforeColon } if key == elemName { - return newJsonTreeImporterInternal(key, v) + return newJsonTreeImporterElement(key, v) } } } return nil } -func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { +func (j *JsonTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { var result []importer.ImportConfigAdapterElement switch d := j.data.(type) { case map[string]any: @@ -64,10 +85,10 @@ func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { switch subElem := v.(type) { case []any: for _, listElem := range subElem { - result = append(result, newJsonTreeImporterInternal(key, listElem)) + result = append(result, newJsonTreeImporterElement(key, listElem)) } default: - result = append(result, newJsonTreeImporterInternal(key, v)) + result = append(result, newJsonTreeImporterElement(key, v)) } } default: @@ -76,17 +97,18 @@ func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { return result } -func (j *JsonTreeImporter) GetKeyValue() (string, error) { +func (j *JsonTreeImporterElement) GetKeyValue() (string, error) { return fmt.Sprintf("%v", j.data), nil } -func (j *JsonTreeImporter) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { +func (j *JsonTreeImporterElement) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { return utils.ConvertJsonValueToTv(j.data, slt) } -func (j *JsonTreeImporter) GetName() string { +func (j *JsonTreeImporterElement) GetName() string { return j.name } // Function to ensure JsonTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*JsonTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*JsonTreeImporterElement)(nil) diff --git a/pkg/tree/importer/json/json_tree_importer_test.go b/pkg/tree/importer/json/json_tree_importer_test.go index 527eceae..34d0778e 100644 --- a/pkg/tree/importer/json/json_tree_importer_test.go +++ b/pkg/tree/importer/json/json_tree_importer_test.go @@ -20,7 +20,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { name string fields fields args args - want importer.ImportConfigAdapter + want importer.ImportConfigAdapterElement }{ { name: "one", @@ -32,7 +32,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { "foo": "bar", }, }, - want: &JsonTreeImporter{ + want: &JsonTreeImporterElement{ data: "bar", name: "foo", }, @@ -40,7 +40,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := NewJsonTreeImporter(tt.fields.data) + j := NewJsonTreeImporter(tt.fields.data, "owner1", 5, false) if got := j.GetElement(tt.args.key); !reflect.DeepEqual(got, tt.want) { t.Errorf("JsonTreeImporter.GetElement() = %v, want %v", got, tt.want) } @@ -77,7 +77,7 @@ func TestJsonTreeImporter_GetKeyValue(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := newJsonTreeImporterInternal(tt.fields.name, tt.fields.data) + j := newJsonTreeImporterElement(tt.fields.name, tt.fields.data) if got, _ := j.GetKeyValue(); got != tt.want { t.Errorf("JsonTreeImporter.GetKeyValue() = %v, want %v", got, tt.want) @@ -106,7 +106,7 @@ func TestJsonTreeImporter_GetName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := &JsonTreeImporter{ + j := &JsonTreeImporterElement{ data: tt.fields.data, name: tt.fields.name, } @@ -164,7 +164,7 @@ func TestJsonTreeImporter_GetTVValue(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := &JsonTreeImporter{ + j := &JsonTreeImporterElement{ data: tt.fields.data, name: tt.fields.name, } diff --git a/pkg/tree/importer/proto/proto_tree_importer.go b/pkg/tree/importer/proto/proto_tree_importer.go index 544cd3c3..7ad01a19 100644 --- a/pkg/tree/importer/proto/proto_tree_importer.go +++ b/pkg/tree/importer/proto/proto_tree_importer.go @@ -12,7 +12,10 @@ import ( type ProtoTreeImporter struct { ProtoTreeImporterElement - deletes *sdcpb.PathSet + deletes *sdcpb.PathSet + intentName string + priority int32 + nonRevertive bool } func NewProtoTreeImporter(data *tree_persist.Intent) *ProtoTreeImporter { @@ -23,10 +26,29 @@ func NewProtoTreeImporter(data *tree_persist.Intent) *ProtoTreeImporter { ProtoTreeImporterElement: ProtoTreeImporterElement{ data: data.GetRoot(), }, - deletes: pathSet, + deletes: pathSet, + intentName: data.GetIntentName(), + priority: data.GetPriority(), + nonRevertive: data.GetNonRevertive(), } } +func (p *ProtoTreeImporter) GetPriority() int32 { + return p.priority +} + +func (p *ProtoTreeImporter) GetNonRevertive() bool { + return p.nonRevertive +} + +func (p *ProtoTreeImporter) GetName() string { + return p.intentName +} + +func (p *ProtoTreeImporter) GetDeletes() *sdcpb.PathSet { + return p.deletes +} + type ProtoTreeImporterElement struct { data *tree_persist.TreeElement } @@ -37,10 +59,6 @@ func NewProtoTreeImporterElement(data *tree_persist.TreeElement) *ProtoTreeImpor } } -func (p *ProtoTreeImporter) GetDeletes() *sdcpb.PathSet { - return p.deletes -} - func (p *ProtoTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { if p.data == nil || len(p.data.Childs) == 0 { return nil @@ -83,3 +101,4 @@ func (p *ProtoTreeImporterElement) GetName() string { // Function to ensure ProtoTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*ProtoTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*ProtoTreeImporterElement)(nil) diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index a762d33f..910f3797 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/google/go-cmp/cmp" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" jimport "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" @@ -112,7 +114,7 @@ func TestProtoTreeImporter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tc := tree.NewTreeContext(scb, "test") + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -126,8 +128,10 @@ func TestProtoTreeImporter(t *testing.T) { t.Fatalf("error parsing json document: %v", err) } - jti := jimport.NewJsonTreeImporter(j) - err = root.ImportConfig(ctx, nil, jti, "owner1", 5, types.NewUpdateInsertFlags()) + jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatal(err) } @@ -138,14 +142,14 @@ func TestProtoTreeImporter(t *testing.T) { } t.Log(root.String()) - protoIntent, err := root.TreeExport("owner1", 5, false) + protoIntent, err := root.TreeExport("owner1", 5) if err != nil { t.Error(err) } fmt.Println(protoIntent.PrettyString(" ")) - tcNew := tree.NewTreeContext(scb, "test") + tcNew := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) rootNew, err := tree.NewTreeRoot(ctx, tcNew) if err != nil { t.Error(err) @@ -153,7 +157,8 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, protoIntent.GetIntentName(), protoIntent.GetPriority(), types.NewUpdateInsertFlags()) + vpf2 := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vpf2) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer.go b/pkg/tree/importer/xml/xml_tree_importer.go index 3ca4dc36..f2c5d0ad 100644 --- a/pkg/tree/importer/xml/xml_tree_importer.go +++ b/pkg/tree/importer/xml/xml_tree_importer.go @@ -9,28 +9,56 @@ import ( ) type XmlTreeImporter struct { - elem *etree.Element + *XmlTreeImporterElement + intentName string + priority int32 + nonRevertive bool } -func NewXmlTreeImporter(d *etree.Element) *XmlTreeImporter { +func NewXmlTreeImporter(d *etree.Element, intentName string, priority int32, nonRevertive bool) *XmlTreeImporter { return &XmlTreeImporter{ + XmlTreeImporterElement: NewXmlTreeImporterElement(d), + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, + } +} + +func (x *XmlTreeImporter) GetName() string { + return x.intentName +} + +func (x *XmlTreeImporter) GetPriority() int32 { + return x.priority +} + +func (x *XmlTreeImporter) GetNonRevertive() bool { + return x.nonRevertive +} + +type XmlTreeImporterElement struct { + elem *etree.Element +} + +func NewXmlTreeImporterElement(d *etree.Element) *XmlTreeImporterElement { + return &XmlTreeImporterElement{ elem: d, } } -func (x *XmlTreeImporter) GetDeletes() *sdcpb.PathSet { +func (x *XmlTreeImporterElement) GetDeletes() *sdcpb.PathSet { return sdcpb.NewPathSet() } -func (x *XmlTreeImporter) GetElement(key string) importer.ImportConfigAdapterElement { +func (x *XmlTreeImporterElement) GetElement(key string) importer.ImportConfigAdapterElement { e := x.elem.FindElement(key) if e == nil { return nil } - return NewXmlTreeImporter(e) + return NewXmlTreeImporterElement(e) } -func (x *XmlTreeImporter) GetElements() []importer.ImportConfigAdapterElement { +func (x *XmlTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { childs := x.elem.ChildElements() if len(childs) == 0 { return nil @@ -39,23 +67,24 @@ func (x *XmlTreeImporter) GetElements() []importer.ImportConfigAdapterElement { result := make([]importer.ImportConfigAdapterElement, 0, len(childs)) for _, c := range childs { - result = append(result, NewXmlTreeImporter(c)) + result = append(result, NewXmlTreeImporterElement(c)) } return result } -func (x *XmlTreeImporter) GetKeyValue() (string, error) { +func (x *XmlTreeImporterElement) GetKeyValue() (string, error) { return x.elem.Text(), nil } -func (x *XmlTreeImporter) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { +func (x *XmlTreeImporterElement) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { return sdcpb.TVFromString(slt, x.elem.Text(), 0) } -func (x *XmlTreeImporter) GetName() string { +func (x *XmlTreeImporterElement) GetName() string { return x.elem.Tag } // Function to ensure JsonTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*XmlTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*XmlTreeImporterElement)(nil) diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index 4ace5325..10cf987f 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -3,10 +3,12 @@ package xml import ( "context" "fmt" + "runtime" "testing" "github.com/beevik/etree" "github.com/google/go-cmp/cmp" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" @@ -77,7 +79,7 @@ func TestXmlTreeImporter(t *testing.T) { } ctx := context.Background() - tc := tree.NewTreeContext(scb, "test") + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -92,7 +94,12 @@ func TestXmlTreeImporter(t *testing.T) { t.Fatal(err) } - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element), "owner1", 5, types.NewUpdateInsertFlags()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + + _, err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), sharedPool) + sharedPool.CloseForSubmit() + sharedPool.Wait() + if err != nil { t.Fatal(err) } diff --git a/pkg/tree/json.go b/pkg/tree/json.go index 23abe03b..151f2f24 100644 --- a/pkg/tree/json.go +++ b/pkg/tree/json.go @@ -4,6 +4,7 @@ import ( "fmt" "slices" + "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) @@ -41,7 +42,7 @@ func (s *sharedEntryAttributes) toJsonInternal(onlyNewOrUpdated bool, ietf bool) // ancestor is a list with keys. result := map[string]any{} - for key, c := range s.GetChilds(DescendMethodActiveChilds) { + for key, c := range s.GetChilds(types.DescendMethodActiveChilds) { ancest, _ := s.GetFirstAncestorWithSchema() prefixedKey := jsonGetIetfPrefixConditional(key, c, ancest, ietf) // recurse the call @@ -101,7 +102,7 @@ func (s *sharedEntryAttributes) toJsonInternal(onlyNewOrUpdated bool, ietf bool) default: // otherwise this is a map result := map[string]any{} - for key, c := range s.GetChilds(DescendMethodActiveChilds) { + for key, c := range s.GetChilds(types.DescendMethodActiveChilds) { prefixedKey := jsonGetIetfPrefixConditional(key, c, s, ietf) js, err := c.toJsonInternal(onlyNewOrUpdated, ietf) if err != nil { diff --git a/pkg/tree/json_test.go b/pkg/tree/json_test.go index 51d04c09..007ccab6 100644 --- a/pkg/tree/json_test.go +++ b/pkg/tree/json_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -378,7 +380,7 @@ func TestToJsonTable(t *testing.T) { ctx := context.Background() - tc := NewTreeContext(scb, owner) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/leaf_entry.go b/pkg/tree/leaf_entry.go index 30856fb1..8abab2ba 100644 --- a/pkg/tree/leaf_entry.go +++ b/pkg/tree/leaf_entry.go @@ -69,6 +69,8 @@ func (l *LeafEntry) MarkNew() { } func (l *LeafEntry) RemoveDeleteFlag() *LeafEntry { + l.mu.Lock() + defer l.mu.Unlock() l.Delete = false return l } @@ -137,9 +139,19 @@ func (l *LeafEntry) MarkExpliciteDelete() { l.IsNew = false } +func (l *LeafEntry) NonRevertive() bool { + l.mu.RLock() + defer l.mu.RUnlock() + // this is a hack that makes the tests pass + if l.parentEntry == nil { + return false + } + return l.parentEntry.GetTreeContext().IsNonRevertiveIntent(l.Owner()) +} + // String returns a string representation of the LeafEntry func (l *LeafEntry) String() string { - return fmt.Sprintf("Owner: %s, Priority: %d, Value: %s, New: %t, Delete: %t, Update: %t, DeleteIntendedOnly: %t, ExplicitDelete: %t", l.Owner(), l.Priority(), l.Value().ToString(), l.GetNewFlag(), l.GetDeleteFlag(), l.GetUpdateFlag(), l.GetDeleteOnlyIntendedFlag(), l.GetExplicitDeleteFlag()) + return fmt.Sprintf("Owner: %s, Priority: %d, Value: %s, New: %t, Delete: %t, Update: %t, DeleteIntendedOnly: %t, ExplicitDelete: %t, Non-Revertive: %t", l.Owner(), l.Priority(), l.Value().ToString(), l.GetNewFlag(), l.GetDeleteFlag(), l.GetUpdateFlag(), l.GetDeleteOnlyIntendedFlag(), l.GetExplicitDeleteFlag(), l.NonRevertive()) } // Compare used for slices.SortFunc. Sorts by path and if equal paths then by owner as the second criteria diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index 0a7a3abd..d3d8f17a 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -25,7 +25,7 @@ func newLeafVariants(tc *TreeContext, parentEnty Entry) *LeafVariants { } } -func (lv *LeafVariants) Add(le *LeafEntry) { +func (lv *LeafVariants) AddWithStats(le *LeafEntry, stats *types.ImportStats) { if leafVariant := lv.GetByOwner(le.Owner()); leafVariant != nil { if leafVariant.Update.Equal(le.Update) { // it seems like the element was not deleted, so drop the delete flag @@ -33,15 +33,22 @@ func (lv *LeafVariants) Add(le *LeafEntry) { } else { // if a leafentry of the same owner exists with different value, mark it for update leafVariant.MarkUpdate(le.Update) + stats.IncrementUpdated() + } } else { lv.lesMutex.Lock() defer lv.lesMutex.Unlock() // if LeafVaraint with same owner does not exist, add the new entry lv.les = append(lv.les, le) + stats.IncrementNew() } } +func (lv *LeafVariants) Add(le *LeafEntry) { + lv.AddWithStats(le, nil) +} + // Items iterator for the LeafVariants func (lv *LeafVariants) Items() iter.Seq[*LeafEntry] { return func(yield func(*LeafEntry) bool) { @@ -89,6 +96,20 @@ func (lv *LeafVariants) canDeleteBranch(keepDefault bool) bool { return true } +// RemoveDeletedByOwner removes and returns the LeafEntry owned by the given owner if it is marked for deletion. +func (lv *LeafVariants) RemoveDeletedByOwner(owner string) *LeafEntry { + lv.lesMutex.Lock() + defer lv.lesMutex.Unlock() + for i, l := range lv.les { + if l.Owner() == owner && l.GetDeleteFlag() { + // Remove element from slice + lv.les = append(lv.les[:i], lv.les[i+1:]...) + return l + } + } + return nil +} + // checkOnlyRunningAndMaybeDefault checks if only running and maybe default LeafVariants exist func (lv *LeafVariants) checkOnlyRunningAndMaybeDefault() bool { lv.lesMutex.RLock() @@ -294,7 +315,10 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau return nil } + // figure out the highest precedence LeafEntry var highest *LeafEntry + // the second highests is the backup in case the highest is marked for deletion + // so this is not actually the second highest always, but the next candidate var secondHighest *LeafEntry for _, e := range lv.les { // first entry set result to it @@ -311,13 +335,13 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau highest = e } else { // check if the update is at least higher prio (lower number) then the secondHighest - if secondHighest == nil || secondHighest.Priority() > e.Priority() { + if secondHighest == nil || secondHighest.Priority() > e.Priority() && !e.GetDeleteFlag() { secondHighest = e } } } - if highest.IsExplicitDelete && !includeExplicitDelete { + if highest == nil || highest.IsExplicitDelete && !includeExplicitDelete { return nil } @@ -348,14 +372,19 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau return nil } +// highestIsUnequalRunning checks if the highest precedence LeafEntry is unequal to the running LeafEntry +// Expects the caller to hold the read lock on lesMutex. func (lv *LeafVariants) highestIsUnequalRunning(highest *LeafEntry) bool { - lv.lesMutex.RLock() - defer lv.lesMutex.RUnlock() // if highes is already running or even default, return false if highest.Update.Owner() == RunningIntentName { return false } + // if highest is not new or updated and highest is non-revertive + if !highest.IsNew && !highest.IsUpdated && lv.tc.IsNonRevertiveIntent(highest.Update.Owner()) { + return false + } + runVal := lv.GetByOwner(RunningIntentName) if runVal == nil { return true @@ -392,6 +421,29 @@ func (lv *LeafVariants) MarkOwnerForDeletion(owner string, onlyIntended bool) *L return nil } +func (lv *LeafVariants) ResetFlags(deleteFlag bool, newFlag bool, updatedFlag bool) int { + lv.lesMutex.Lock() + defer lv.lesMutex.Unlock() + count := 0 + + for _, le := range lv.les { + if deleteFlag && le.Delete { + le.Delete = false + le.DeleteOnlyIntended = false + count++ + } + if updatedFlag && le.IsUpdated { + le.IsUpdated = false + count++ + } + if newFlag && le.IsNew { + le.IsNew = false + count++ + } + } + return count +} + func (lv *LeafVariants) DeleteByOwner(owner string) *LeafEntry { lv.lesMutex.Lock() defer lv.lesMutex.Unlock() diff --git a/pkg/tree/parallelImporter.go b/pkg/tree/parallelImporter.go deleted file mode 100644 index 83f7ee2d..00000000 --- a/pkg/tree/parallelImporter.go +++ /dev/null @@ -1,173 +0,0 @@ -package tree - -import ( - "context" - "fmt" - "slices" - "sync" - - "github.com/sdcio/data-server/pkg/pool" - "github.com/sdcio/data-server/pkg/tree/importer" - "github.com/sdcio/data-server/pkg/tree/types" - sdcpb "github.com/sdcio/sdc-protos/sdcpb" - "google.golang.org/protobuf/types/known/emptypb" -) - -type importTask struct { - entry Entry - importerElement importer.ImportConfigAdapterElement - intentName string - intentPrio int32 - insertFlags *types.UpdateInsertFlags - treeContext *TreeContext - leafListLock *sync.Map -} - -func (s *sharedEntryAttributes) ImportConfig( - ctx context.Context, - importerElement importer.ImportConfigAdapterElement, - intentName string, - intentPrio int32, - insertFlags *types.UpdateInsertFlags, -) error { - p := pool.NewWorkerPool[importTask](ctx, 1) - - p.Start(importHandler) - - // seed root - if err := p.Submit(importTask{entry: s, importerElement: importerElement, intentName: intentName, intentPrio: intentPrio, insertFlags: insertFlags, treeContext: s.treeContext, leafListLock: &sync.Map{}}); err != nil { - return err - } - - // signal we are done seeding external tasks (workers may still submit) - p.CloseForSubmit() - - // wait for the import to finish (or error) - return p.Wait() -} - -func importHandler(ctx context.Context, task importTask, submit func(importTask) error) error { - - elem := task.entry.PathName() - _ = elem - - switch x := task.entry.GetSchema().GetSchema().(type) { - case *sdcpb.SchemaElem_Container, nil: - // keyed container: handle keys sequentially - if len(task.entry.GetSchema().GetContainer().GetKeys()) > 0 { - var exists bool - var actual Entry = task.entry - var keyChild Entry - - keys := task.entry.GetSchemaKeys() - slices.Sort(keys) - for _, k := range keys { - ktrans := task.importerElement.GetElement(k) - if ktrans == nil { - return fmt.Errorf("unable to find key attribute %s under %s", k, task.entry.SdcpbPath().ToXPath(false)) - } - kv, err := ktrans.GetKeyValue() - if err != nil { - return err - } - if keyChild, exists = actual.GetChild(kv); !exists { - keyChild, err = newEntry(ctx, actual, kv, task.treeContext) - if err != nil { - return err - } - } - actual = keyChild - } - // submit resolved entry with same adapter element - // return importHandler(ctx, importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext}, submit) - return submit(importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}) - } - - // presence container or children - elems := task.importerElement.GetElements() - if len(elems) == 0 { - schem := task.entry.GetSchema().GetContainer() - if schem != nil && schem.IsPresence { - tv := &sdcpb.TypedValue{Value: &sdcpb.TypedValue_EmptyVal{EmptyVal: &emptypb.Empty{}}} - upd := types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) - task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.insertFlags, task.entry)) - } - return nil - } - - // submit each child (no external locking per your guarantee) - for _, childElt := range elems { - child, exists := task.entry.GetChild(childElt.GetName()) - if !exists { - var err error - child, err = newEntry(ctx, task.entry, childElt.GetName(), task.treeContext) - if err != nil { - return fmt.Errorf("error inserting %s at %s: %w", childElt.GetName(), task.entry.SdcpbPath().ToXPath(false), err) - } - } - if err := submit(importTask{entry: child, importerElement: childElt, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}); err != nil { - return err - } - } - return nil - - case *sdcpb.SchemaElem_Field: - tv, err := task.importerElement.GetTVValue(ctx, x.Field.GetType()) - if err != nil { - return err - } - upd := types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) - task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.insertFlags, task.entry)) - return nil - - case *sdcpb.SchemaElem_Leaflist: - // for the leaflist, since in XML the leaf list elements are independet elements, we need to make - // sure that the first element is basically resetting the leaflist and all consecutive elemts are then - // added to the already resettet leaflist. - // strategy here is to create a mutex lock it and try to store it in the leafListLock map. - // if the mutex was then stored, we're the first goroutine and need to reset. If we get a different mutex back - // and the the loaded var is set to true, we should not reset the list and trxy to lock the returned mutex. - - // create a mutex and lock it - llMutex := &sync.Mutex{} - llMutex.Lock() - - // try storing it or load it from leafListLock - llm, loaded := task.leafListLock.LoadOrStore(task.entry.SdcpbPath().ToXPath(false), llMutex) - - // if it was loaded, we need to lock the loaded mutex - if loaded { - llMutex = llm.(*sync.Mutex) - llMutex.Lock() - } - defer llMutex.Unlock() - - var scalarArr *sdcpb.ScalarArray - mustAdd := false - var le *LeafEntry - if loaded { - le = task.entry.GetLeafVariantEntries().GetByOwner(task.intentName) - scalarArr = le.Value().GetLeaflistVal() - } else { - le = NewLeafEntry(nil, task.insertFlags, task.entry) - mustAdd = true - scalarArr = &sdcpb.ScalarArray{Element: []*sdcpb.TypedValue{}} - } - - tv, err := task.importerElement.GetTVValue(ctx, x.Leaflist.GetType()) - if err != nil { - return err - } - if tv.GetLeaflistVal() == nil { - scalarArr.Element = append(scalarArr.Element, tv) - tv = &sdcpb.TypedValue{Value: &sdcpb.TypedValue_LeaflistVal{LeaflistVal: scalarArr}} - } - le.Update = types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) - if mustAdd { - task.entry.GetLeafVariantEntries().Add(le) - } - return nil - default: - return nil - } -} diff --git a/pkg/tree/processor_blame_config.go b/pkg/tree/processor_blame_config.go index 90063420..603b4ca7 100644 --- a/pkg/tree/processor_blame_config.go +++ b/pkg/tree/processor_blame_config.go @@ -2,9 +2,10 @@ package tree import ( "context" - "sync" + "errors" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/protobuf/proto" ) @@ -29,32 +30,26 @@ func NewBlameConfigProcessorConfig(includeDefaults bool) *BlameConfigProcessorCo } } +// Run processes the entry tree starting from e, building a blame tree showing which owner +// (intent) is responsible for each configuration value. The pool parameter should be +// VirtualFailFast to stop on first error. +// Returns the blame tree structure and any error encountered. func (p *BlameConfigProcessor) Run(ctx context.Context, e Entry, pool pool.VirtualPoolI) (*sdcpb.BlameTreeElement, error) { - dropChan := make(chan *DropBlameChild, 10) - - wg := &sync.WaitGroup{} - wg.Add(1) - // execute the deletes in a seperate single channel - go func(dC <-chan *DropBlameChild) { - for elem := range dC { - elem.Exec() - } - wg.Done() - }(dropChan) - blameTask := NewBlameConfigTask(e, dropChan, p.config) - err := pool.Submit(blameTask) - if err != nil { + blameTask := NewBlameConfigTask(e, p.config) + if err := pool.Submit(blameTask); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() return nil, err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() - // close the dropChan channel - close(dropChan) - wg.Wait() + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + if errs := pool.Errors(); len(errs) > 0 { + return blameTask.self, errors.Join(errs...) + } return blameTask.self, pool.FirstError() } @@ -63,16 +58,14 @@ type BlameConfigTask struct { parent *sdcpb.BlameTreeElement self *sdcpb.BlameTreeElement selfEntry Entry - dropChan chan<- *DropBlameChild } -func NewBlameConfigTask(e Entry, dropChan chan<- *DropBlameChild, c *BlameConfigProcessorConfig) *BlameConfigTask { +func NewBlameConfigTask(e Entry, c *BlameConfigProcessorConfig) *BlameConfigTask { return &BlameConfigTask{ config: c, parent: nil, self: &sdcpb.BlameTreeElement{}, selfEntry: e, - dropChan: dropChan, } } @@ -99,25 +92,30 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) t.self.SetDeviationValue(runningLe.Value()) } } - } else { - // if it is default but no default is meant to be returned - t.dropChan <- &DropBlameChild{parent: t.parent, dropElem: t.self} } } - for _, childEntry := range t.selfEntry.GetChilds(DescendMethodActiveChilds) { + childs := t.selfEntry.GetChilds(types.DescendMethodActiveChilds) + for _, childKey := range childs.SortedKeys() { + childEntry := childs[childKey] + childHighestLe := childEntry.GetLeafVariantEntries().GetHighestPrecedence(false, true, true) + if childHighestLe != nil { + if childHighestLe.Update.Owner() == DefaultsIntentName && !t.config.includeDefaults { + continue + } + } + child := &sdcpb.BlameTreeElement{Name: childEntry.PathName()} t.self.AddChild(child) - // create a new task for each child + // Create a new task for each child task := &BlameConfigTask{ config: t.config, parent: t.self, self: child, selfEntry: childEntry, - dropChan: t.dropChan, } - // submit the task + // Submit may fail if pool is closed or fail-fast error occurred if err := submit(task); err != nil { return err } @@ -125,22 +123,3 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) return nil } - -type DropBlameChild struct { - parent *sdcpb.BlameTreeElement - dropElem *sdcpb.BlameTreeElement -} - -func (d *DropBlameChild) Exec() { - // from parent drop the child dropElem - index := -1 - for i, child := range d.parent.GetChilds() { - if child == d.dropElem { - index = i - break - } - } - if index != -1 { - d.parent.Childs = append(d.parent.Childs[:index], d.parent.Childs[index+1:]...) - } -} diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index e85a628a..75f2aee2 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -39,14 +39,14 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -71,14 +71,14 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -104,20 +104,20 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } conf2 := config2() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -132,7 +132,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } @@ -152,8 +152,8 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { treeRoot := tt.r(t) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - vPool := sharedPool.NewVirtualPool(pool.VirtualFailFast, 10) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + vPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) bp := NewBlameConfigProcessor(NewBlameConfigProcessorConfig(tt.includeDefaults)) got, err := bp.Run(ctx, treeRoot.sharedEntryAttributes, vPool) diff --git a/pkg/tree/processor_error_collection_test.go b/pkg/tree/processor_error_collection_test.go new file mode 100644 index 00000000..a0cb142f --- /dev/null +++ b/pkg/tree/processor_error_collection_test.go @@ -0,0 +1,107 @@ +package tree + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/sdcio/data-server/pkg/pool" +) + +// TestProcessorErrorCollection verifies that processor error collection happens +// AFTER all tasks complete, not before. This ensures CloseAndWait is called +// before checking pool.Errors(). +func TestProcessorErrorCollection(t *testing.T) { + // Create a shared task pool + ctx := context.Background() + sharedPool := pool.NewSharedTaskPool(ctx, 2) + + // Test with tolerant mode to collect multiple errors + vpool := sharedPool.NewVirtualPool(pool.VirtualTolerant) + + var errorCount atomic.Int32 + var taskCompletionCount atomic.Int32 + + // Submit tasks that will error with a delay + // This simulates tasks that take time to complete and add errors + for i := 0; i < 5; i++ { + err := vpool.Submit(pool.TaskFunc(func(ctx context.Context, submit func(pool.Task) error) error { + // Simulate some work + time.Sleep(10 * time.Millisecond) + taskCompletionCount.Add(1) + errorCount.Add(1) + return errors.New("task error") + })) + if err != nil { + t.Fatalf("Failed to submit task: %v", err) + } + } + + // Close and wait - this MUST complete before we check errors + vpool.CloseAndWait() + + // Now verify all errors are collected + errs := vpool.Errors() + + // All 5 tasks should have completed + if count := taskCompletionCount.Load(); count != 5 { + t.Errorf("Expected 5 tasks to complete, got %d", count) + } + + // All 5 errors should be collected + if len(errs) != 5 { + t.Errorf("Expected 5 errors, got %d", len(errs)) + } + + // Verify combined error works + combinedErr := errors.Join(errs...) + if combinedErr == nil { + t.Error("Expected combined error to be non-nil") + } + + sharedPool.CloseForSubmit() + sharedPool.Wait() +} + +// TestProcessorEarlyReturnCleanup verifies that if Submit fails early, +// the pool is still properly cleaned up. +func TestProcessorEarlyReturnCleanup(t *testing.T) { + ctx := context.Background() + sharedPool := pool.NewSharedTaskPool(ctx, 2) + + vpool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + + // Close the pool immediately so Submit will fail + vpool.CloseForSubmit() + + // Try to submit - this should fail + err := vpool.Submit(pool.TaskFunc(func(ctx context.Context, submit func(pool.Task) error) error { + return nil + })) + + if err == nil { + t.Fatal("Expected Submit to fail on closed pool") + } + + // Even though Submit failed, we should be able to call CloseAndWait safely + vpool.CloseAndWait() + + // Wait should not block + done := make(chan struct{}) + go func() { + vpool.Wait() + close(done) + }() + + select { + case <-done: + // Success - Wait unblocked + case <-time.After(1 * time.Second): + t.Fatal("Wait blocked after CloseAndWait on failed Submit") + } + + sharedPool.CloseForSubmit() + sharedPool.Wait() +} diff --git a/pkg/tree/processor_explicit_delete.go b/pkg/tree/processor_explicit_delete.go new file mode 100644 index 00000000..afc601a5 --- /dev/null +++ b/pkg/tree/processor_explicit_delete.go @@ -0,0 +1,120 @@ +package tree + +import ( + "context" + "sync" + + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils" +) + +type ExplicitDeleteProcessor struct { + params *ExplicitDeleteTaskParameters +} + +func NewExplicitDeleteProcessor(params *ExplicitDeleteTaskParameters) *ExplicitDeleteProcessor { + return &ExplicitDeleteProcessor{ + params: params, + } +} + +// GetExplicitDeleteCreationCount returns the amount of all the explicitDelete LeafVariants that where created. +func (edp *ExplicitDeleteProcessor) GetExplicitDeleteCreationCount() int { + return len(edp.params.relatedLeafVariants) +} + +// GetCreatedExplicitDeleteLeafEntries returns all the explicitDelete LeafVariants that where created. +func (edp *ExplicitDeleteProcessor) GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice { + return edp.params.relatedLeafVariants +} + +type ExplicitDeleteTaskParameters struct { + owner string + priority int32 + relatedLeafVariants []*LeafEntry + rlvMutex *sync.Mutex +} + +func NewExplicitDeleteTaskParameters(owner string, priority int32) *ExplicitDeleteTaskParameters { + return &ExplicitDeleteTaskParameters{ + priority: priority, + owner: owner, + relatedLeafVariants: []*LeafEntry{}, + rlvMutex: &sync.Mutex{}, + } +} + +func (p *ExplicitDeleteProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { + taskpool := poolFactory.NewVirtualPool(pool.VirtualTolerant) + err := taskpool.Submit(newExplicitDeleteTask(e, p.params)) + taskpool.CloseAndWait() + return err +} + +type explicitDeleteTask struct { + entry Entry + params *ExplicitDeleteTaskParameters +} + +func newExplicitDeleteTask(entry Entry, params *ExplicitDeleteTaskParameters) *explicitDeleteTask { + return &explicitDeleteTask{ + entry: entry, + params: params, + } +} + +func (t *explicitDeleteTask) Run(ctx context.Context, submit func(pool.Task) error) error { + if t.entry.HoldsLeafvariants() { + le := t.entry.GetLeafVariantEntries().GetByOwner(t.params.owner) + if le != nil { + le.MarkExpliciteDelete() + } else { + le = t.entry.GetLeafVariantEntries().AddExplicitDeleteEntry(t.params.owner, t.params.priority) + } + t.params.rlvMutex.Lock() + t.params.relatedLeafVariants = append(t.params.relatedLeafVariants, le) + t.params.rlvMutex.Unlock() + } + + // trigger the execution on all childs + for _, c := range t.entry.GetChilds(types.DescendMethodAll) { + err := submit(newExplicitDeleteTask(c, t.params)) + if err != nil { + return err + } + } + + return nil +} + +// Stats structs +type ExplicitDeleteProcessorStat interface { + GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice + GetExplicitDeleteCreationCount() int +} + +type ExplicitDeleteProcessorStatCollection map[string]ExplicitDeleteProcessorStat + +func (e ExplicitDeleteProcessorStatCollection) Stats() map[string]int { + return utils.MapApplyFuncToMap(e, func(k string, v ExplicitDeleteProcessorStat) int { + return v.GetExplicitDeleteCreationCount() + }) +} + +func (e ExplicitDeleteProcessorStatCollection) Count() int { + count := 0 + for _, stat := range e { + count += stat.GetExplicitDeleteCreationCount() + } + return count +} + +func (e ExplicitDeleteProcessorStatCollection) ContainsEntries() bool { + for _, stat := range e { + if stat.GetExplicitDeleteCreationCount() > 0 { + return true + } + } + return false +} diff --git a/pkg/tree/visitor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go similarity index 87% rename from pkg/tree/visitor_explicit_delete_test.go rename to pkg/tree/processor_explicit_delete_test.go index 172b44cb..2b3939a8 100644 --- a/pkg/tree/visitor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -2,11 +2,13 @@ package tree import ( "context" + "runtime" "strings" "testing" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" @@ -46,14 +48,14 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) if err != nil { t.Error(err) } @@ -74,19 +76,19 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -126,29 +128,29 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } - testhelper.LoadYgotStructIntoTreeRoot(ctx, &sdcio_schema.Device{Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + loadYgotStructIntoTreeRoot(ctx, &sdcio_schema.Device{Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { Name: ygot.String("ethernet-1/1"), Description: ygot.String("mydesc"), }, - }}, root, owner2, owner2Prio, flagsNew) + }}, root, owner2, owner2Prio, false, flagsNew) return root }, @@ -276,7 +278,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root := tt.root() - root.AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) + root.GetTreeContext().AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) err := root.FinishInsertionPhase(ctx) if err != nil { diff --git a/pkg/tree/processor_importer.go b/pkg/tree/processor_importer.go new file mode 100644 index 00000000..e2cc8ac1 --- /dev/null +++ b/pkg/tree/processor_importer.go @@ -0,0 +1,224 @@ +package tree + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/sdcio/data-server/pkg/pool" + + treeimporter "github.com/sdcio/data-server/pkg/tree/importer" + "github.com/sdcio/data-server/pkg/tree/types" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "google.golang.org/protobuf/types/known/emptypb" +) + +type importConfigTask struct { + entry Entry + importerElement treeimporter.ImportConfigAdapterElement + params *ImportConfigProcessorParams +} + +type ImportConfigProcessorParams struct { + intentName string + intentPrio int32 + insertFlags *types.UpdateInsertFlags + treeContext *TreeContext + leafListTracker *sync.Map + stats *types.ImportStats +} + +func NewParameters( + intentName string, + intentPrio int32, + insertFlags *types.UpdateInsertFlags, + treeContext *TreeContext, + leafListLock *sync.Map, + stats *types.ImportStats, +) *ImportConfigProcessorParams { + return &ImportConfigProcessorParams{ + intentName: intentName, + intentPrio: intentPrio, + insertFlags: insertFlags, + treeContext: treeContext, + leafListTracker: leafListLock, + stats: stats, + } +} + +type ImportConfigProcessor struct { + importer treeimporter.ImportConfigAdapter + insertFlags *types.UpdateInsertFlags + stats *types.ImportStats +} + +func NewImportConfigProcessor(importer treeimporter.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { + return &ImportConfigProcessor{ + importer: importer, + insertFlags: insertFlags, + stats: types.NewImportStats(), + } +} + +func (p *ImportConfigProcessor) GetStats() *types.ImportStats { + return p.stats +} + +func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { + // store non revertive info + e.GetTreeContext().nonRevertiveInfo[p.importer.GetName()] = p.importer.GetNonRevertive() + + // store explicit deletes + e.GetTreeContext().explicitDeletes.Add(p.importer.GetName(), p.importer.GetPriority(), p.importer.GetDeletes()) + + workerPool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + + t := importConfigTask{ + entry: e, + importerElement: p.importer, + params: NewParameters(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.GetTreeContext(), &sync.Map{}, p.stats), + } + + if err := workerPool.Submit(t); err != nil { + workerPool.CloseAndWait() + return err + } + + workerPool.CloseAndWait() + + if err := workerPool.FirstError(); err != nil { + return err + } + return nil +} + +func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) error) error { + + elem := task.entry.PathName() + _ = elem + + switch x := task.entry.GetSchema().GetSchema().(type) { + case *sdcpb.SchemaElem_Container, nil: + // keyed container: handle keys sequentially + if len(task.entry.GetSchema().GetContainer().GetKeys()) > 0 { + var exists bool + var actual Entry = task.entry + var keyChild Entry + + keys := task.entry.GetSchemaKeys() + slices.Sort(keys) + for _, k := range keys { + ktrans := task.importerElement.GetElement(k) + if ktrans == nil { + return fmt.Errorf("unable to find key attribute %s under %s", k, task.entry.SdcpbPath().ToXPath(false)) + } + kv, err := ktrans.GetKeyValue() + if err != nil { + return err + } + if keyChild, exists = actual.GetChild(kv); !exists { + keyChild, err = NewEntry(ctx, actual, kv, task.params.treeContext) + if err != nil { + return err + } + } + actual = keyChild + } + // submit resolved entry with same adapter element + // return importHandler(ctx, importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext}, submit) + return submit(importConfigTask{entry: actual, importerElement: task.importerElement, params: task.params}) + } + + // presence container or children + elems := task.importerElement.GetElements() + if len(elems) == 0 { + schem := task.entry.GetSchema().GetContainer() + if schem != nil && schem.IsPresence { + tv := &sdcpb.TypedValue{Value: &sdcpb.TypedValue_EmptyVal{EmptyVal: &emptypb.Empty{}}} + upd := types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) + task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.params.insertFlags, task.entry)) + } + return nil + } + + // submit each child + for _, childElt := range elems { + child, exists := task.entry.GetChild(childElt.GetName()) + if !exists { + var err error + child, err = NewEntry(ctx, task.entry, childElt.GetName(), task.params.treeContext) + if err != nil { + return fmt.Errorf("error inserting %s at %s: %w", childElt.GetName(), task.entry.SdcpbPath().ToXPath(false), err) + } + } + // need to process Leaflist childs in this goroutine to avois reordering + switch child.GetSchema().GetSchema().(type) { + case *sdcpb.SchemaElem_Leaflist: + err := importConfigTask{entry: child, importerElement: childElt, params: task.params}.Run(ctx, submit) + if err != nil { + return err + } + default: + if err := submit(importConfigTask{entry: child, importerElement: childElt, params: task.params}); err != nil { + return err + } + } + } + return nil + + case *sdcpb.SchemaElem_Field: + tv, err := task.importerElement.GetTVValue(ctx, x.Field.GetType()) + if err != nil { + return err + } + upd := types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) + task.entry.GetLeafVariantEntries().AddWithStats(NewLeafEntry(upd, task.params.insertFlags, task.entry), task.params.stats) + return nil + + case *sdcpb.SchemaElem_Leaflist: + // For leaf lists we need to make sure the first insertion resets the leaf list, all consecutive insertions do add to it. + // To do so, we have the leafListTracker map that indicates if a leaf list was already reset and the first insertion was done or not. + // The key for the map is the combination of the parent entry and the leaf list name, so we can have multiple leaf lists under the same parent without + // interference. + + // create a unique key for the leaflist based on the parent entry and the leaflist name + key := struct { + parent Entry + name string + }{task.entry.GetParent(), task.importerElement.GetName()} + + _, loaded := task.params.leafListTracker.LoadOrStore(key, struct{}{}) + + var scalarArr *sdcpb.ScalarArray + mustAdd := false + var le *LeafEntry + if loaded { + // if loaded is true, it means that another goroutine already did the first insertion and reset, + // so we just need to get the leaf list and add to it + le = task.entry.GetLeafVariantEntries().GetByOwner(task.params.intentName) + scalarArr = le.Value().GetLeaflistVal() + } else { + // reset / create the leaf list on the first insertion + le = NewLeafEntry(nil, task.params.insertFlags, task.entry) + mustAdd = true + scalarArr = &sdcpb.ScalarArray{Element: []*sdcpb.TypedValue{}} + } + + tv, err := task.importerElement.GetTVValue(ctx, x.Leaflist.GetType()) + if err != nil { + return err + } + if tv.GetLeaflistVal() == nil { + scalarArr.Element = append(scalarArr.Element, tv) + tv = &sdcpb.TypedValue{Value: &sdcpb.TypedValue_LeaflistVal{LeaflistVal: scalarArr}} + } + le.Update = types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) + if mustAdd { + task.entry.GetLeafVariantEntries().Add(le) + } + return nil + default: + return nil + } +} diff --git a/pkg/tree/processor_mark_owner_delete.go b/pkg/tree/processor_mark_owner_delete.go index 2dcf5c49..ffe93abf 100644 --- a/pkg/tree/processor_mark_owner_delete.go +++ b/pkg/tree/processor_mark_owner_delete.go @@ -2,9 +2,11 @@ package tree import ( "context" + "errors" "sync" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" ) type MarkOwnerDeleteProcessor struct { @@ -19,17 +21,24 @@ func NewOwnerDeleteMarker(c *OwnerDeleteMarkerTaskConfig) *MarkOwnerDeleteProces } } -func (o *MarkOwnerDeleteProcessor) Run(e Entry, pool pool.VirtualPoolI) error { - - err := pool.Submit(newOwnerDeleteMarkerTask(o.config, e, o.matches)) - if err != nil { +// Run processes the entry tree starting from e, marking leaf variant entries for deletion +// by the specified owner. The pool parameter should be VirtualFailFast to stop on first error. +// Returns the first error encountered, or nil if successful. +func (p *MarkOwnerDeleteProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + if err := pool.Submit(newOwnerDeleteMarkerTask(p.config, e, p.matches)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() return err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + if errs := pool.Errors(); len(errs) > 0 { + return errors.Join(errs...) + } return pool.FirstError() } @@ -67,8 +76,12 @@ func (x ownerDeleteMarkerTask) Run(ctx context.Context, submit func(pool.Task) e if le != nil { x.matches.Append(le) } - for _, c := range x.e.GetChilds(DescendMethodAll) { - submit(newOwnerDeleteMarkerTask(x.config, c, x.matches)) + // Process children recursively + for _, c := range x.e.GetChilds(types.DescendMethodAll) { + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(newOwnerDeleteMarkerTask(x.config, c, x.matches)); err != nil { + return err + } } return nil } diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go new file mode 100644 index 00000000..0827b31c --- /dev/null +++ b/pkg/tree/processor_remove_deleted.go @@ -0,0 +1,115 @@ +package tree + +import ( + "context" + "errors" + "sync" + "sync/atomic" + + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" +) + +type RemoveDeletedProcessor struct { + config *RemoveDeletedProcessorParameters +} + +func NewRemoveDeletedProcessor(c *RemoveDeletedProcessorParameters) *RemoveDeletedProcessor { + return &RemoveDeletedProcessor{ + config: c, + } +} + +type RemoveDeletedProcessorParameters struct { + owner string + deleteStatsCount atomic.Int64 + zeroLeafEntryElements []Entry + zeroLeafEntryElementsLock sync.Mutex +} + +func NewRemoveDeletedProcessorParameters(owner string) *RemoveDeletedProcessorParameters { + return &RemoveDeletedProcessorParameters{ + owner: owner, + deleteStatsCount: atomic.Int64{}, + zeroLeafEntryElements: []Entry{}, + zeroLeafEntryElementsLock: sync.Mutex{}, + } +} + +func (r *RemoveDeletedProcessorParameters) GetDeleteStatsCount() int64 { + return r.deleteStatsCount.Load() +} + +// GetZeroLengthLeafVariantEntries returns the entries that have zero-length leaf variant entries after removal +func (r *RemoveDeletedProcessorParameters) GetZeroLengthLeafVariantEntries() []Entry { + r.zeroLeafEntryElementsLock.Lock() + defer r.zeroLeafEntryElementsLock.Unlock() + return r.zeroLeafEntryElements +} + +// Run processes the entry tree starting from e, removing leaf variant entries marked +// for deletion by the specified owner. The pool parameter should be VirtualFailFast +// to stop on first error. +// Returns the first error encountered, or nil if successful. +func (p *RemoveDeletedProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { + + // create a virtual task pool for removeDeleted operations + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + + if err := pool.Submit(newRemoveDeletedTask(p.config, e, false)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() + return err + } + + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + return errors.Join(pool.Errors()...) +} + +type removeDeletedTask struct { + config *RemoveDeletedProcessorParameters + e Entry + keepDefaults bool +} + +func newRemoveDeletedTask(c *RemoveDeletedProcessorParameters, e Entry, keepDefaults bool) *removeDeletedTask { + return &removeDeletedTask{ + config: c, + e: e, + keepDefaults: keepDefaults, + } +} + +func (t *removeDeletedTask) Run(ctx context.Context, submit func(pool.Task) error) error { + if ctx.Err() != nil { + return ctx.Err() + } + + res := t.e.GetLeafVariantEntries().RemoveDeletedByOwner(t.config.owner) + if res != nil { + // increment the delete stats count + t.config.deleteStatsCount.Add(1) + } + if t.e.CanDeleteBranch(t.keepDefaults) { + func() { + t.config.zeroLeafEntryElementsLock.Lock() + defer t.config.zeroLeafEntryElementsLock.Unlock() + t.config.zeroLeafEntryElements = append(t.config.zeroLeafEntryElements, t.e) + }() + return nil + } + + // Process children recursively + for _, c := range t.e.GetChilds(types.DescendMethodAll) { + childTask := newRemoveDeletedTask(t.config, c, t.e.GetSchema().GetContainer() == nil) + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(childTask); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go new file mode 100644 index 00000000..623fb410 --- /dev/null +++ b/pkg/tree/processor_reset_flags.go @@ -0,0 +1,107 @@ +package tree + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" +) + +// ResetFlagsProcessor resets the flags on leaf variant entries +type ResetFlagsProcessor struct { + config *ResetFlagsProcessorParameters +} + +func NewResetFlagsProcessor(c *ResetFlagsProcessorParameters) *ResetFlagsProcessor { + return &ResetFlagsProcessor{ + config: c, + } +} + +type ResetFlagsProcessorParameters struct { + deleteFlag, newFlag, updateFlag bool + adjustedFlagsCount atomic.Int64 +} + +func NewResetFlagsProcessorParameters() *ResetFlagsProcessorParameters { + return &ResetFlagsProcessorParameters{ + adjustedFlagsCount: atomic.Int64{}, + } +} + +func (r *ResetFlagsProcessorParameters) SetDeleteFlag() *ResetFlagsProcessorParameters { + r.deleteFlag = true + return r +} + +func (r *ResetFlagsProcessorParameters) SetNewFlag() *ResetFlagsProcessorParameters { + r.newFlag = true + return r +} + +func (r *ResetFlagsProcessorParameters) SetUpdateFlag() *ResetFlagsProcessorParameters { + r.updateFlag = true + return r +} + +// GetAdjustedFlagsCount returns the number of flags that were adjusted +func (r *ResetFlagsProcessorParameters) GetAdjustedFlagsCount() int64 { + return r.adjustedFlagsCount.Load() +} + +// Run processes the entry tree starting from e, resetting flags on all leaf variant entries +// according to the processor configuration. The pool parameter can be either VirtualFailFast +// (stops on first error) or VirtualTolerant (collects all errors). +// Returns the first error for fail-fast pools, or a combined error for tolerant pools. +func (p *ResetFlagsProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { + if e == nil { + return fmt.Errorf("entry cannot be nil") + } + + // create a virtual task pool for resetFlags operations + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + + // Submit root task; workers will recursively process children + if err := pool.Submit(newResetFlagsTask(p.config, e)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() + return err + } + + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + return errors.Join(pool.Errors()...) +} + +type resetFlagsTask struct { + config *ResetFlagsProcessorParameters + e Entry +} + +func newResetFlagsTask(config *ResetFlagsProcessorParameters, e Entry) *resetFlagsTask { + return &resetFlagsTask{ + config: config, + e: e, + } +} + +func (t *resetFlagsTask) Run(ctx context.Context, submit func(pool.Task) error) error { + // Reset flags as per config + count := t.e.GetLeafVariantEntries().ResetFlags(t.config.deleteFlag, t.config.newFlag, t.config.updateFlag) + t.config.adjustedFlagsCount.Add(int64(count)) + + // Process children recursively + for _, c := range t.e.GetChilds(types.DescendMethodAll) { + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(newResetFlagsTask(t.config, c)); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go new file mode 100644 index 00000000..424f4b17 --- /dev/null +++ b/pkg/tree/processor_reset_flags_test.go @@ -0,0 +1,133 @@ +package tree + +import ( + "context" + "fmt" + "runtime" + "testing" + + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResetFlagsProcessorRun tests the Run method of ResetFlagsProcessor +func TestResetFlagsProcessorRun(t *testing.T) { + tests := []struct { + name string + deleteFlag bool + newFlag bool + updateFlag bool + wantErr bool + tree func() *RootEntry + adjustCount int64 + }{ + { + name: "successful reset with all flags", + deleteFlag: true, + newFlag: true, + updateFlag: true, + wantErr: false, + adjustCount: 4, + tree: func() *RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + + root, err := NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + _, err = root.AddUpdateRecursive(ctx, + &sdcpb.Path{Elem: []*sdcpb.PathElem{ + sdcpb.NewPathElem("interface", map[string]string{"name": "ethernet-1/1"}), + sdcpb.NewPathElem("description", nil)}, + }, + types.NewUpdate(nil, testhelper.GetStringTvProto("desc 1/1"), RunningValuesPrio, RunningIntentName, 0), + flagsNew, + ) + if err != nil { + t.Fatalf("failed to add update recursive: %v", err) + } + + _, err = root.AddUpdateRecursive(ctx, + &sdcpb.Path{Elem: []*sdcpb.PathElem{ + sdcpb.NewPathElem("interface", map[string]string{"name": "ethernet-1/2"}), + sdcpb.NewPathElem("description", nil)}, + }, + types.NewUpdate(nil, testhelper.GetStringTvProto("desc 1/2"), RunningValuesPrio, RunningIntentName, 0), + types.NewUpdateInsertFlags().SetDeleteFlag(), + ) + if err != nil { + t.Fatalf("failed to add update recursive: %v", err) + } + + err = root.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + + return root + + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + // Create processor parameters + params := NewResetFlagsProcessorParameters() + + if tt.deleteFlag { + params.SetDeleteFlag() + } + if tt.newFlag { + params.SetNewFlag() + } + if tt.updateFlag { + params.SetUpdateFlag() + } + + // Create processor + processor := NewResetFlagsProcessor(params) + require.NotNil(t, processor) + + // Create a mock entry for testing + // Note: In a real test, you would use a properly initialized Entry + // This is a simplified version - you may need to adjust based on your actual Entry implementation + + // Create a virtual pool for testing + taskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + + root := tt.tree() + + fmt.Println(root.String()) + + processorErr := processor.Run(root.GetRoot(), taskPool) + if (processorErr != nil) != tt.wantErr { + t.Errorf("ResetFlagsProcessor.Run() error = %v, wantErr %v", processorErr, tt.wantErr) + return + } + + fmt.Println(root.String()) + + // For now, we'll test that the processor can be created and run method exists + // Full integration testing would require properly mocked Entry objects + assert.NotNil(t, processor) + assert.Equal(t, int64(tt.adjustCount), params.GetAdjustedFlagsCount()) + }) + } +} diff --git a/pkg/tree/processor_validate.go b/pkg/tree/processor_validate.go index a8a018c5..314d3d91 100644 --- a/pkg/tree/processor_validate.go +++ b/pkg/tree/processor_validate.go @@ -19,10 +19,9 @@ func NewValidateProcessor(parameters *ValidateProcessorParameters) *ValidateProc } func (p *ValidateProcessor) Run(taskpoolFactory pool.VirtualPoolFactory, e Entry) { - taskpool := taskpoolFactory.NewVirtualPool(pool.VirtualTolerant, 0) + taskpool := taskpoolFactory.NewVirtualPool(pool.VirtualTolerant) taskpool.Submit(newValidateTask(e, p.parameters)) - taskpool.CloseForSubmit() - taskpool.Wait() + taskpool.CloseAndWait() } type ValidateProcessorParameters struct { @@ -56,9 +55,9 @@ func (t *validateTask) Run(ctx context.Context, submit func(pool.Task) error) er return nil } // validate the mandatory statement on this entry - if t.e.remainsToExist() { + if t.e.RemainsToExist() { t.e.ValidateLevel(ctx, t.parameters.resultChan, t.parameters.stats, t.parameters.vCfg) - for _, c := range t.e.GetChilds(DescendMethodActiveChilds) { + for _, c := range t.e.GetChilds(types.DescendMethodActiveChilds) { submit(newValidateTask(c, t.parameters)) } } diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 8eea8a8f..02e937c2 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -20,7 +20,6 @@ import ( // RootEntry the root of the cache.Update tree type RootEntry struct { *sharedEntryAttributes - explicitDeletes *DeletePathSet } var ( @@ -36,12 +35,6 @@ func NewTreeRoot(ctx context.Context, tc *TreeContext) (*RootEntry, error) { root := &RootEntry{ sharedEntryAttributes: sea, - explicitDeletes: NewDeletePaths(), - } - - err = tc.SetRoot(sea) - if err != nil { - return nil, err } return root, nil @@ -54,7 +47,7 @@ func (r *RootEntry) stringToDisk(filename string) error { } func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { - tc := r.treeContext.deepCopy() + tc := r.GetTreeContext().deepCopy() se, err := r.sharedEntryAttributes.deepCopy(tc, nil) if err != nil { return nil, err @@ -62,20 +55,11 @@ func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { result := &RootEntry{ sharedEntryAttributes: se, - explicitDeletes: r.explicitDeletes.DeepCopy(), } - err = tc.SetRoot(result.sharedEntryAttributes) - if err != nil { - return nil, err - } return result, nil } -func (r *RootEntry) RemoveExplicitDeletes(intentName string) *sdcpb.PathSet { - return r.explicitDeletes.RemoveIntentDeletes(intentName) -} - func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAndUpdate, flags *types.UpdateInsertFlags) error { var err error for idx, u := range us { @@ -88,21 +72,21 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error { - r.treeContext.SetActualOwner(intentName) - +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) { e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) if err != nil { - return err + return nil, err } - - r.explicitDeletes.Add(intentName, intentPrio, importer.GetDeletes()) - - return e.ImportConfig(ctx, importer, intentName, intentPrio, flags) + ImportConfigProcessor := NewImportConfigProcessor(importer, flags) + err = ImportConfigProcessor.Run(ctx, e, poolFactory) + if err != nil { + return nil, err + } + return ImportConfigProcessor.GetStats(), nil } -func (r *RootEntry) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { - r.explicitDeletes.Add(intentName, priority, pathset) +func (r *RootEntry) SetNonRevertiveIntent(intentName string, nonRevertive bool) { + r.GetTreeContext().nonRevertiveInfo[intentName] = nonRevertive } func (r *RootEntry) Validate(ctx context.Context, vCfg *config.Validation, taskpoolFactory pool.VirtualPoolFactory) (types.ValidationResults, *types.ValidationStats) { @@ -184,13 +168,13 @@ func (r *RootEntry) GetDeviations(ctx context.Context, ch chan<- *types.Deviatio r.sharedEntryAttributes.GetDeviations(ctx, ch, true) } -func (r *RootEntry) TreeExport(owner string, priority int32, deviation bool) (*tree_persist.Intent, error) { +func (r *RootEntry) TreeExport(owner string, priority int32) (*tree_persist.Intent, error) { treeExport, err := r.sharedEntryAttributes.TreeExport(owner) if err != nil { return nil, err } - explicitDeletes := r.explicitDeletes.GetByIntentName(owner).ToPathSlice() + explicitDeletes := r.GetTreeContext().explicitDeletes.GetByIntentName(owner).ToPathSlice() var rootExportEntry *tree_persist.TreeElement if len(treeExport) != 0 { @@ -202,7 +186,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32, deviation bool) (*t IntentName: owner, Root: rootExportEntry, Priority: priority, - Deviation: deviation, + NonRevertive: r.GetTreeContext().nonRevertiveInfo[owner], ExplicitDeletes: explicitDeletes, }, nil } @@ -243,31 +227,35 @@ func (r *RootEntry) DeleteBranchPaths(ctx context.Context, deletes types.DeleteE func (r *RootEntry) FinishInsertionPhase(ctx context.Context) error { log := logf.FromContext(ctx) - edvs := ExplicitDeleteVisitors{} + edpsc := ExplicitDeleteProcessorStatCollection{} // apply the explicit deletes - for deletePathPrio := range r.explicitDeletes.Items() { - edv := NewExplicitDeleteVisitor(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) + for deletePathPrio := range r.GetTreeContext().explicitDeletes.Items() { + + params := NewExplicitDeleteTaskParameters(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) for path := range deletePathPrio.PathItems() { + // set the priority // navigate to the stated path entry, err := r.NavigateSdcpbPath(ctx, path) if err != nil { log.Error(nil, "Applying explicit delete - path not found, skipping", "severity", "WARN", "path", path.ToXPath(false)) } - - // walk the whole branch adding the explicit delete leafvariant - err = entry.Walk(ctx, edv) + edp := NewExplicitDeleteProcessor(params) + err = edp.Run(ctx, entry, r.GetTreeContext().GetPoolFactory()) if err != nil { return err } - edvs[deletePathPrio.GetOwner()] = edv + edpsc[deletePathPrio.GetOwner()] = edp } } - log.V(logf.VDebug).Info("ExplicitDeletes added", "explicit-deletes", utils.MapToString(edvs.Stats(), ", ", func(k string, v int) string { - return fmt.Sprintf("%s=%d", k, v) - })) + // conditional logging + if edpsc.ContainsEntries() { + log.V(logf.VDebug).Info("ExplicitDeletes added", "explicit-deletes", utils.MapToString(edpsc.Stats(), ", ", func(k string, v int) string { + return fmt.Sprintf("%s=%d", k, v) + })) + } return r.sharedEntryAttributes.FinishInsertionPhase(ctx) } diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index ce4d0d05..6303e2bd 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "runtime" "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -24,7 +26,7 @@ import ( func TestRootEntry_TreeExport(t *testing.T) { owner1 := "owner1" owner2 := "owner2" - tc := NewTreeContext(nil, owner1) + tc := NewTreeContext(nil, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) type args struct { owner string @@ -47,6 +49,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -95,6 +98,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -161,6 +165,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -261,6 +266,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -301,9 +307,8 @@ func TestRootEntry_TreeExport(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := &RootEntry{ sharedEntryAttributes: tt.sharedEntryAttributes(), - explicitDeletes: NewDeletePaths(), } - got, err := r.TreeExport(tt.args.owner, tt.args.priority, false) + got, err := r.TreeExport(tt.args.owner, tt.args.priority) if (err != nil) != tt.wantErr { t.Fatalf("RootEntry.TreeExport() error = %v, wantErr %v", err, tt.wantErr) return @@ -392,14 +397,14 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } @@ -429,7 +434,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) type fields struct { sharedEntryAttributes func(t *testing.T) *sharedEntryAttributes @@ -516,7 +521,9 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } - err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny), "owner1", 5, types.NewUpdateInsertFlags()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + ImportConfigProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags()) + err = ImportConfigProcessor.Run(ctx, s, vpf) if err != nil { t.Fatal(err) } @@ -561,16 +568,16 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { { name: "One", rootEntry: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) if err != nil { t.Fatal(err) } @@ -582,12 +589,12 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { }, owner: owner1, want: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } @@ -604,7 +611,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { root := tt.rootEntry(t) got := root.GetUpdatesForOwner(tt.owner).ToPathAndUpdateSlice() - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) resultRoot, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -627,7 +634,6 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { t.Logf("Want:\n%s", wantStr) t.Logf("Got:\n%s", resultRoot.String()) } - }) } } diff --git a/pkg/tree/sharedEntryAttributes.go b/pkg/tree/sharedEntryAttributes.go index 6b281c86..6f8e68bb 100644 --- a/pkg/tree/sharedEntryAttributes.go +++ b/pkg/tree/sharedEntryAttributes.go @@ -122,6 +122,10 @@ func (s *sharedEntryAttributes) GetRoot() Entry { return s.parent.GetRoot() } +func (s *sharedEntryAttributes) GetTreeContext() *TreeContext { + return s.treeContext +} + // loadDefaults helper to populate defaults on the initializiation of the sharedEntryAttribute func (s *sharedEntryAttributes) loadDefaults(ctx context.Context) error { @@ -176,7 +180,7 @@ func (s *sharedEntryAttributes) GetDeviations(ctx context.Context, ch chan<- *ty // if s is a presence container but has active childs, it should not be treated as a presence // container, hence the leafvariants should not be processed. For presence container with // childs the TypedValue.empty_val in the presence container is irrelevant. - if s.schema.GetContainer().GetIsPresence() && len(s.GetChilds(DescendMethodActiveChilds)) > 0 { + if s.schema.GetContainer().GetIsPresence() && len(s.GetChilds(types.DescendMethodActiveChilds)) > 0 { evalLeafvariants = false } @@ -186,7 +190,7 @@ func (s *sharedEntryAttributes) GetDeviations(ctx context.Context, ch chan<- *ty } // get all active childs - activeChilds := s.GetChilds(DescendMethodActiveChilds) + activeChilds := s.GetChilds(types.DescendMethodActiveChilds) // iterate through all childs for cName, c := range s.getChildren() { @@ -256,14 +260,14 @@ func (s *sharedEntryAttributes) checkAndCreateKeysAsLeafs(ctx context.Context, i } if !entryExists { // create a new entry - child, err = newEntry(ctx, s, k.Name, s.treeContext) + child, err = NewEntry(ctx, s, k.Name, s.treeContext) if err != nil { return err } } // add the new child entry to s - err = s.addChild(ctx, child) + err = s.AddChild(ctx, child) if err != nil { return err } @@ -353,7 +357,7 @@ func (s *sharedEntryAttributes) GetListChilds() ([]Entry, error) { for level := 0; level < len(keys); level++ { for _, e := range actualEntries { // add all children - for _, c := range e.GetChilds(DescendMethodAll) { + for _, c := range e.GetChilds(types.DescendMethodAll) { newEntries = append(newEntries, c) } } @@ -390,7 +394,7 @@ func (s *sharedEntryAttributes) FilterChilds(keys map[string]string) ([]Entry, e // therefor we need to go through the processEntries List // and collect all the matching childs for _, entry := range processEntries { - childs := entry.GetChilds(DescendMethodAll) + childs := entry.GetChilds(types.DescendMethodAll) matchEntry, childExists := childs[keyVal] // so if such child, that matches the given filter value exists, we append it to the results if childExists { @@ -401,7 +405,7 @@ func (s *sharedEntryAttributes) FilterChilds(keys map[string]string) ([]Entry, e // this is basically the wildcard case, so go through all childs and add them result = []Entry{} for _, entry := range processEntries { - childs := entry.GetChilds(DescendMethodAll) + childs := entry.GetChilds(types.DescendMethodAll) for _, v := range childs { // hence we add all the existing childs to the result list result = append(result, v) @@ -441,26 +445,6 @@ func (s *sharedEntryAttributes) GetLevel() int { return l } -// Walk takes the EntryVisitor and applies it to every Entry in the tree -func (s *sharedEntryAttributes) Walk(ctx context.Context, v EntryVisitor) error { - - // execute the function locally - err := v.Visit(ctx, s) - if err != nil { - return err - } - - // trigger the execution on all childs - for _, c := range s.GetChilds(v.DescendMethod()) { - err := c.Walk(ctx, v) - if err != nil { - return err - } - } - v.Up() - return nil -} - func (s *sharedEntryAttributes) HoldsLeafvariants() bool { switch x := s.schema.GetSchema().(type) { case *sdcpb.SchemaElem_Container: @@ -554,7 +538,7 @@ func (s *sharedEntryAttributes) canDelete() bool { } // handle containers - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { canDelete := c.canDelete() if !canDelete { s.cacheCanDelete = utils.BoolPtr(false) @@ -565,7 +549,7 @@ func (s *sharedEntryAttributes) canDelete() bool { return *s.cacheCanDelete } -func (s *sharedEntryAttributes) canDeleteBranch(keepDefault bool) bool { +func (s *sharedEntryAttributes) CanDeleteBranch(keepDefault bool) bool { s.cacheMutex.Lock() defer s.cacheMutex.Unlock() @@ -576,7 +560,7 @@ func (s *sharedEntryAttributes) canDeleteBranch(keepDefault bool) bool { // handle containers for _, c := range s.childs.Items() { - canDelete := c.canDeleteBranch(keepDefault) + canDelete := c.CanDeleteBranch(keepDefault) if !canDelete { return false } @@ -601,7 +585,7 @@ func (s *sharedEntryAttributes) shouldDelete() bool { // but a real delete should only be added if there is at least one shouldDelete() == true shouldDelete := false - activeChilds := s.GetChilds(DescendMethodActiveChilds) + activeChilds := s.GetChilds(types.DescendMethodActiveChilds) // if we have no active childs, we can and should delete. if len(s.choicesResolvers) > 0 && len(activeChilds) == 0 { canDelete = true @@ -637,7 +621,7 @@ func (s *sharedEntryAttributes) shouldDelete() bool { return result } -func (s *sharedEntryAttributes) remainsToExist() bool { +func (s *sharedEntryAttributes) RemainsToExist() bool { // see if we have the value cached s.cacheMutex.Lock() defer s.cacheMutex.Unlock() @@ -648,8 +632,8 @@ func (s *sharedEntryAttributes) remainsToExist() bool { // handle containers childsRemain := false - for _, c := range s.GetChilds(DescendMethodActiveChilds) { - childsRemain = c.remainsToExist() + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { + childsRemain = c.RemainsToExist() if childsRemain { break } @@ -740,8 +724,8 @@ func (s *sharedEntryAttributes) String() string { return s.SdcpbPath().ToXPath(false) } -// addChild add an entry to the list of child entries for the entry. -func (s *sharedEntryAttributes) addChild(ctx context.Context, e Entry) error { +// AddChild add an entry to the list of child entries for the entry. +func (s *sharedEntryAttributes) AddChild(ctx context.Context, e Entry) error { // make sure Entry should not only hold LeafEntries if s.leafVariants.Length() > 0 { // An exception are presence containers @@ -781,7 +765,7 @@ func (s *sharedEntryAttributes) NavigateSdcpbPath(ctx context.Context, path *sdc } return entry.NavigateSdcpbPath(ctx, path.CopyAndRemoveFirstPathElem()) default: - e, exists := s.GetChilds(DescendMethodActiveChilds)[pathElems[0].Name] + e, exists := s.GetChilds(types.DescendMethodActiveChilds)[pathElems[0].Name] if !exists { pth := &sdcpb.Path{Elem: pathElems} e, err = s.tryLoadingDefault(ctx, pth) @@ -855,7 +839,7 @@ func (s *sharedEntryAttributes) DeleteBranch(ctx context.Context, path *sdcpb.Pa // which is, forwarding entry to entry.GetParent() as a last step and depending on the remains // return continuing to perform the delete forther up in the tree // with remains initially set to false, we initially call DeleteSubtree on the referenced entry. - for entry.canDeleteBranch(false) { + for entry.CanDeleteBranch(false) { // forward the entry pointer to the parent // depending on the remains var the DeleteSubtree is again called on that parent entry entry = entry.GetParent() @@ -865,16 +849,16 @@ func (s *sharedEntryAttributes) DeleteBranch(ctx context.Context, path *sdcpb.Pa } // calling DeleteSubtree with the empty string, because it should not delete the owner from the higher level keys, // but what it will also do is delete possibly dangling key elements in the tree - entry.deleteCanDeleteChilds(true) + entry.DeleteCanDeleteChilds(true) } return nil } -func (s *sharedEntryAttributes) deleteCanDeleteChilds(keepDefault bool) { +func (s *sharedEntryAttributes) DeleteCanDeleteChilds(keepDefault bool) { // otherwise check all for childname, child := range s.childs.Items() { - if child.canDeleteBranch(keepDefault) { + if child.CanDeleteBranch(keepDefault) { s.childs.DeleteChild(childname) } } @@ -890,7 +874,7 @@ func (s *sharedEntryAttributes) deleteBranchInternal(ctx context.Context, owner if err != nil { return err } - if child.canDeleteBranch(false) { + if child.CanDeleteBranch(false) { s.childs.DeleteChild(childName) } } @@ -907,7 +891,7 @@ func (s *sharedEntryAttributes) GetHighestPrecedence(result LeafVariantSlice, on } // continue with childs. Childs are part of choices, process only the "active" (highes precedence) childs - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { result = c.GetHighestPrecedence(result, onlyNewOrUpdated, includeDefaults, includeExplicitDelete) } return result @@ -978,7 +962,7 @@ func (s *sharedEntryAttributes) getHighestPrecedenceValueOfBranch(filter Highest // it will multiplex all the different Validations that need to happen func (s *sharedEntryAttributes) ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { // validate the mandatory statement on this entry - if s.remainsToExist() { + if s.RemainsToExist() { // TODO: Validate Enums if !vCfg.DisabledValidators.Mandatory { s.validateMandatory(ctx, resultChan, stats) @@ -1115,7 +1099,7 @@ func (s *sharedEntryAttributes) validateMinMaxElements(resultChan chan<- *types. ownersSet := map[string]struct{}{} for _, child := range childs { - childAttributes := child.GetChilds(DescendMethodActiveChilds) + childAttributes := child.GetChilds(types.DescendMethodActiveChilds) keyName := contSchema.GetKeys()[0].GetName() if keyAttr, ok := childAttributes[keyName]; ok { highestPrec := keyAttr.GetHighestPrecedence(nil, false, false, false) @@ -1230,7 +1214,7 @@ func (s *sharedEntryAttributes) validatePattern(resultChan chan<- *types.Validat // defined by the schema are present either in the tree or in the index. func (s *sharedEntryAttributes) validateMandatory(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats) { log := logf.FromContext(ctx) - if !s.remainsToExist() { + if !s.RemainsToExist() { return } if s.schema != nil { @@ -1293,9 +1277,9 @@ func (s *sharedEntryAttributes) validateMandatoryWithKeys(ctx context.Context, l // iterate over the attributes make sure any of these exists for _, attr := range attributes { // first check if the mandatory value is set via the intent, e.g. part of the tree already - v, existsInTree = s.GetChilds(DescendMethodActiveChilds)[attr] + v, existsInTree = s.GetChilds(types.DescendMethodActiveChilds)[attr] // if exists and remains to Exist - if existsInTree && v.remainsToExist() { + if existsInTree && v.RemainsToExist() { // set success to true and break the loop success = true break @@ -1316,7 +1300,7 @@ func (s *sharedEntryAttributes) validateMandatoryWithKeys(ctx context.Context, l return } - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { c.validateMandatoryWithKeys(ctx, level-1, attributes, choiceName, resultChan) } } @@ -1369,7 +1353,7 @@ func (s *sharedEntryAttributes) FinishInsertionPhase(ctx context.Context) error // recurse the call to all (active) entries within the tree. // Thereby already using the choiceCaseResolver via filterActiveChoiceCaseChilds() - for _, child := range s.GetChilds(DescendMethodActiveChilds) { + for _, child := range s.GetChilds(types.DescendMethodActiveChilds) { err = child.FinishInsertionPhase(ctx) if err != nil { return err @@ -1435,15 +1419,15 @@ func (s *sharedEntryAttributes) GetChild(name string) (Entry, bool) { return s.childs.GetEntry(name) } -func (s *sharedEntryAttributes) GetChilds(d DescendMethod) EntryMap { +func (s *sharedEntryAttributes) GetChilds(d types.DescendMethod) EntryMap { if s.schema == nil { return s.childs.GetAll() } switch d { - case DescendMethodAll: + case types.DescendMethodAll: return s.childs.GetAll() - case DescendMethodActiveChilds: + case types.DescendMethodActiveChilds: skipAttributesList := s.choicesResolvers.GetSkipElements() // if there are no items that should be skipped, take a shortcut // and simply return all childs straight away @@ -1463,31 +1447,6 @@ func (s *sharedEntryAttributes) GetChilds(d DescendMethod) EntryMap { return nil } -// // filterActiveChoiceCaseChilds returns the list of child elements. In case the Entry is -// // a container with a / multiple choices, the list of childs is filtered to only return the -// // cases that have the highest precedence. -// func (s *sharedEntryAttributes) filterActiveChoiceCaseChilds() map[string]Entry { -// if s.schema == nil { -// return s.childs.GetAll() -// } - -// skipAttributesList := s.choicesResolvers.GetSkipElements() -// // if there are no items that should be skipped, take a shortcut -// // and simply return all childs straight away -// if len(skipAttributesList) == 0 { -// return s.childs.GetAll() -// } -// result := map[string]Entry{} -// // optimization option: sort the slices and forward in parallel, lifts extra burden that the contains call holds. -// for childName, child := range s.childs.GetAll() { -// if slices.Contains(skipAttributesList, childName) { -// continue -// } -// result[childName] = child -// } -// return result -// } - // StringIndent returns the sharedEntryAttributes in its string representation // The string is intented according to the nesting level in the yang model func (s *sharedEntryAttributes) StringIndent(result []string) []string { @@ -1610,14 +1569,14 @@ func (s *sharedEntryAttributes) getOrCreateChilds(ctx context.Context, path *sdc var current Entry = s for i, pe := range path.Elem { // Step 1: Find or create the child for the path element name - newCurrent, exists := current.GetChilds(DescendMethodAll)[pe.Name] + newCurrent, exists := current.GetChilds(types.DescendMethodAll)[pe.Name] if !exists { var err error - child, err := newEntry(ctx, current, pe.Name, s.treeContext) + child, err := NewEntry(ctx, current, pe.Name, s.treeContext) if err != nil { return nil, err } - if err := current.addChild(ctx, child); err != nil { + if err := current.AddChild(ctx, child); err != nil { return nil, err } newCurrent = child @@ -1633,14 +1592,14 @@ func (s *sharedEntryAttributes) getOrCreateChilds(ctx context.Context, path *sdc // Step 2: For each key, find or create the key child for _, key := range keys { - newCurrent, exists = current.GetChilds(DescendMethodAll)[pe.Key[key]] + newCurrent, exists = current.GetChilds(types.DescendMethodAll)[pe.Key[key]] if !exists { var err error - keyChild, err := newEntry(ctx, current, pe.Key[key], s.treeContext) + keyChild, err := NewEntry(ctx, current, pe.Key[key], s.treeContext) if err != nil { return nil, err } - if err := current.addChild(ctx, keyChild); err != nil { + if err := current.AddChild(ctx, keyChild); err != nil { return nil, err } newCurrent = keyChild @@ -1691,12 +1650,12 @@ func (s *sharedEntryAttributes) addUpdateRecursiveInternal(ctx context.Context, var x Entry = s var exists bool for name := range path.GetElem()[idx].PathElemNames() { - if e, exists = x.GetChilds(DescendMethodAll)[name]; !exists { - newE, err := newEntry(ctx, x, name, s.treeContext) + if e, exists = x.GetChilds(types.DescendMethodAll)[name]; !exists { + newE, err := NewEntry(ctx, x, name, s.treeContext) if err != nil { return nil, err } - err = x.addChild(ctx, newE) + err = x.AddChild(ctx, newE) if err != nil { return nil, err } diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index 03408f0b..c569cd55 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -37,7 +37,7 @@ func Test_sharedEntryAttributes_checkAndCreateKeysAsLeafs(t *testing.T) { ctx := context.Background() scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -93,7 +93,8 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { { name: "just rootEntry", root: func() *RootEntry { - tc := NewTreeContext(nil, owner1) + tc := NewTreeContext(nil, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) + r := &RootEntry{ sharedEntryAttributes: &sharedEntryAttributes{ pathElemName: "__root__", @@ -103,7 +104,6 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { parent: nil, treeContext: tc, }, - explicitDeletes: NewDeletePaths(), } r.leafVariants = newLeafVariants(tc, r.sharedEntryAttributes) return r @@ -123,7 +123,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -146,7 +146,8 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Error(err) } @@ -203,16 +204,16 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -241,16 +242,16 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -338,13 +339,13 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -406,7 +407,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { for _, elem := range got { elemNames = append(elemNames, elem.PathName()) elemChilds[elem.PathName()] = []string{} - for k := range elem.GetChilds(DescendMethodAll) { + for k := range elem.GetChilds(types.DescendMethodAll) { elemChilds[elem.PathName()] = append(elemChilds[elem.PathName()], k) } } @@ -452,14 +453,14 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -474,7 +475,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } @@ -631,7 +632,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -654,7 +655,8 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -670,7 +672,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MustStatement = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) @@ -753,7 +755,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -776,7 +778,8 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -791,7 +794,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { valConfig := validationConfig.DeepCopy() valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MustStatement = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) @@ -806,7 +809,6 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { ctx := context.TODO() - owner1 := "owner1" tests := []struct { name string @@ -864,7 +866,7 @@ func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -910,14 +912,14 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -942,7 +944,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -974,7 +976,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -1003,7 +1005,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1017,7 +1019,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -1041,7 +1043,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { validationConfig.DisabledValidators.DisableAll() validationConfig.DisabledValidators.Mandatory = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResults, _ := root.Validate(ctx, validationConfig, sharedPool) @@ -1122,7 +1124,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1149,7 +1151,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println(root.String()) - treepersist, err := root.TreeExport(owner1, owner1Prio, false) + treepersist, err := root.TreeExport(owner1, owner1Prio) if err != nil { t.Error(err) return @@ -1163,24 +1165,24 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println("\nTreeExport:") fmt.Println(string(persistByte)) - tcNew := NewTreeContext(scb, owner1) + tcNew := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) newRoot, err := NewTreeRoot(ctx, tcNew) if err != nil { t.Fatal(err) } - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), owner1, owner1Prio, flagsExisting) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vpf) if err != nil { t.Error(err) return } // mark owner delete - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1298,7 +1300,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1321,7 +1323,8 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -1337,7 +1340,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MaxElements = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) t.Log(strings.Join(result.ErrorsStr(), "\n")) @@ -1467,7 +1470,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1490,7 +1493,8 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -1506,7 +1510,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MaxElements = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) t.Log(strings.Join(result.ErrorsStr(), "\n")) diff --git a/pkg/tree/sorter.go b/pkg/tree/sorter.go index 8bbda48b..54a6433d 100644 --- a/pkg/tree/sorter.go +++ b/pkg/tree/sorter.go @@ -1,16 +1,18 @@ package tree +import "github.com/sdcio/data-server/pkg/tree/types" + func getListEntrySortFunc(parent Entry) func(a, b Entry) int { // return the comparison function return func(a, b Entry) int { keys := parent.GetSchemaKeys() var cmpResult int for _, v := range keys { - achild, exists := a.GetChilds(DescendMethodAll)[v] + achild, exists := a.GetChilds(types.DescendMethodAll)[v] if !exists { return 0 } - bchild, exists := b.GetChilds(DescendMethodAll)[v] + bchild, exists := b.GetChilds(types.DescendMethodAll)[v] if !exists { return 0 } diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index 8dc1d19c..e0297de5 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -1,43 +1,61 @@ package tree import ( - "fmt" - schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) type TreeContext struct { - root Entry // the trees root element - schemaClient schemaClient.SchemaClientBound - actualOwner string + schemaClient schemaClient.SchemaClientBound + nonRevertiveInfo map[string]bool + explicitDeletes *DeletePathSet + poolFactory pool.VirtualPoolFactory } -func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *TreeContext { +func NewTreeContext(sc schemaClient.SchemaClientBound, poolFactory pool.VirtualPoolFactory) *TreeContext { return &TreeContext{ - schemaClient: sc, - actualOwner: actualOwner, + schemaClient: sc, + nonRevertiveInfo: map[string]bool{}, + explicitDeletes: NewDeletePaths(), + poolFactory: poolFactory, } } // deepCopy root is required to be set manually func (t *TreeContext) deepCopy() *TreeContext { - return &TreeContext{ + tc := &TreeContext{ schemaClient: t.schemaClient, + poolFactory: t.poolFactory, } -} -func (t *TreeContext) SetRoot(e Entry) error { - if t.root != nil { - return fmt.Errorf("trying to set treecontexts root, although it is already set") + // deepcopy nonRevertiveInfo + m := make(map[string]bool, len(t.nonRevertiveInfo)) + for k, v := range t.nonRevertiveInfo { + m[k] = v } - t.root = e - return nil + tc.nonRevertiveInfo = m + tc.explicitDeletes = t.explicitDeletes.DeepCopy() + return tc +} + +func (t *TreeContext) GetPoolFactory() pool.VirtualPoolFactory { + return t.poolFactory +} + +func (t *TreeContext) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { + t.explicitDeletes.Add(intentName, priority, pathset) +} + +func (t *TreeContext) RemoveExplicitDeletes(intentName string) *sdcpb.PathSet { + return t.explicitDeletes.RemoveIntentDeletes(intentName) } -func (t *TreeContext) GetActualOwner() string { - return t.actualOwner +func (t *TreeContext) AddNonRevertiveInfo(intent string, nonRevertive bool) { + t.nonRevertiveInfo[intent] = nonRevertive } -func (t *TreeContext) SetActualOwner(owner string) { - t.actualOwner = owner +// IsNonRevertiveIntent returns the non-revertive flag per intent. False is also returned the intent does not exist. +func (t *TreeContext) IsNonRevertiveIntent(intent string) bool { + return t.nonRevertiveInfo[intent] } diff --git a/pkg/tree/types/descend_methods.go b/pkg/tree/types/descend_methods.go new file mode 100644 index 00000000..5291ea78 --- /dev/null +++ b/pkg/tree/types/descend_methods.go @@ -0,0 +1,8 @@ +package types + +type DescendMethod int + +const ( + DescendMethodAll DescendMethod = iota + DescendMethodActiveChilds +) diff --git a/pkg/tree/types/import_stats.go b/pkg/tree/types/import_stats.go new file mode 100644 index 00000000..447627dc --- /dev/null +++ b/pkg/tree/types/import_stats.go @@ -0,0 +1,50 @@ +package types + +import ( + "fmt" + "sync/atomic" +) + +type ImportStats struct { + newEntries atomic.Int64 + updatedEntries atomic.Int64 +} + +func NewImportStats() *ImportStats { + return &ImportStats{} +} + +func (is *ImportStats) String() string { + return fmt.Sprintf("NewEntries: %d, UpdatedEntries: %d", is.newEntries.Load(), is.updatedEntries.Load()) +} + +func (is *ImportStats) Join(i *ImportStats) { + is.newEntries.Add(i.newEntries.Load()) + is.updatedEntries.Add(i.updatedEntries.Load()) +} + +func (is *ImportStats) IncrementNew() { + if is == nil { + return + } + is.newEntries.Add(1) +} + +func (is *ImportStats) IncrementUpdated() { + if is == nil { + return + } + is.updatedEntries.Add(1) +} + +func (is *ImportStats) GetNewCount() int64 { + return is.newEntries.Load() +} + +func (is *ImportStats) GetUpdatedCount() int64 { + return is.updatedEntries.Load() +} + +func (is *ImportStats) Changed() bool { + return is.newEntries.Load() > 0 || is.updatedEntries.Load() > 0 +} diff --git a/pkg/tree/utils.go b/pkg/tree/utils.go new file mode 100644 index 00000000..06edda8c --- /dev/null +++ b/pkg/tree/utils.go @@ -0,0 +1,45 @@ +package tree + +import ( + "context" + "encoding/json" + "runtime" + + "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/pkg/pool" + + "github.com/sdcio/data-server/pkg/tree/importer" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + "github.com/sdcio/data-server/pkg/tree/types" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +type RootTreeImport interface { + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) +} + +func loadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root *RootEntry, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) (*types.ImportStats, error) { + jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: true, + }) + if err != nil { + return nil, err + } + + var jsonConfAny any + err = json.Unmarshal([]byte(jconfStr), &jsonConfAny) + if err != nil { + return nil, err + } + + stp := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + + importProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) + err = importProcessor.Run(ctx, root.sharedEntryAttributes, stp) + + if err != nil { + return nil, err + } + return importProcessor.GetStats(), nil +} diff --git a/pkg/tree/validation_entry_leafref.go b/pkg/tree/validation_entry_leafref.go index 3367270f..25a63c91 100644 --- a/pkg/tree/validation_entry_leafref.go +++ b/pkg/tree/validation_entry_leafref.go @@ -61,7 +61,7 @@ func (s *sharedEntryAttributes) BreadthSearch(ctx context.Context, sdcpbPath *sd } // if the entry is will be deleted by the actual operation, skip it. - if !entry.remainsToExist() { + if !entry.RemainsToExist() { continue } // if we're at the final level, no child filtering is needed any more, diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 944aff6d..6ed11249 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -213,7 +215,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -236,7 +238,8 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index bed8495a..7b2d39c7 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -27,7 +27,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) @@ -65,9 +65,10 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) } @@ -78,7 +79,6 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { } valConf := validationConfig.DeepCopy() - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, valConf, sharedPool) @@ -157,7 +157,7 @@ func TestValidate_RangesSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -179,10 +179,13 @@ func TestValidate_RangesSigned(t *testing.T) { } // new json tree importer - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) + + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) + if err != nil { t.Error(err) } @@ -192,7 +195,6 @@ func TestValidate_RangesSigned(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) t.Logf("Validation Errors:\n%s", strings.Join(validationResult.ErrorsStr(), "\n")) @@ -291,7 +293,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -313,10 +315,12 @@ func TestValidate_RangesUnSigned(t *testing.T) { } // new json tree importer - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) + if err != nil { t.Error(err) } @@ -327,7 +331,6 @@ func TestValidate_RangesUnSigned(t *testing.T) { } // run validation - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResults, _ := root.Validate(ctx, validationConfig, sharedPool) t.Logf("Validation Errors:\n%s", strings.Join(validationResults.ErrorsStr(), "\n")) diff --git a/pkg/tree/visitor_base.go b/pkg/tree/visitor_base.go deleted file mode 100644 index 96772a11..00000000 --- a/pkg/tree/visitor_base.go +++ /dev/null @@ -1,12 +0,0 @@ -package tree - -// BaseVisitor abstract base visitor implementation that all the concrete visitory are ment to embed. -type BaseVisitor struct{} - -func (b *BaseVisitor) Up() { - // noop -} - -func (b *BaseVisitor) DescendMethod() DescendMethod { - return DescendMethodAll -} diff --git a/pkg/tree/visitor_explicit_delete.go b/pkg/tree/visitor_explicit_delete.go deleted file mode 100644 index 1c1f5c83..00000000 --- a/pkg/tree/visitor_explicit_delete.go +++ /dev/null @@ -1,125 +0,0 @@ -package tree - -import ( - "context" - "sync" - - "github.com/sdcio/data-server/pkg/utils" -) - -type ExplicitDeleteVisitor struct { - BaseVisitor - owner string - priority int32 - - // created entries for further stat calculation - relatedLeafVariants LeafVariantSlice - rlvMutex *sync.Mutex -} - -var _ EntryVisitor = (*ExplicitDeleteVisitor)(nil) - -func NewExplicitDeleteVisitor(owner string, priority int32) *ExplicitDeleteVisitor { - return &ExplicitDeleteVisitor{ - priority: priority, - owner: owner, - relatedLeafVariants: []*LeafEntry{}, - rlvMutex: &sync.Mutex{}, - } -} - -func (edv *ExplicitDeleteVisitor) Visit(ctx context.Context, e Entry) error { - if !e.HoldsLeafvariants() { - return nil - } - le := e.GetLeafVariantEntries().GetByOwner(edv.owner) - if le != nil { - le.MarkExpliciteDelete() - } else { - le = e.GetLeafVariantEntries().AddExplicitDeleteEntry(edv.owner, edv.priority) - } - edv.rlvMutex.Lock() - edv.relatedLeafVariants = append(edv.relatedLeafVariants, le) - edv.rlvMutex.Unlock() - return nil -} - -// GetExplicitDeleteCreationCount returns the amount of all the explicitDelete LeafVariants that where created. -func (edv *ExplicitDeleteVisitor) GetExplicitDeleteCreationCount() int { - return len(edv.relatedLeafVariants) -} - -// GetCreatedExplicitDeleteLeafEntries returns all the explicitDelete LeafVariants that where created. -func (edv *ExplicitDeleteVisitor) GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice { - return edv.relatedLeafVariants -} - -// ExplicitDeleteVisitors map of *ExplicitDeleteVisitor indexed by the intent name -type ExplicitDeleteVisitors map[string]*ExplicitDeleteVisitor - -func (e ExplicitDeleteVisitors) Stats() map[string]int { - return utils.MapApplyFuncToMap(e, func(k string, v *ExplicitDeleteVisitor) int { - return v.GetExplicitDeleteCreationCount() - }) -} - -// type ExplicitDeleteTask struct { -// e Entry -// owner string -// priority int32 -// } - -// func NewExplicitDeleteTask(e Entry, owner string, priority int32) *ExplicitDeleteTask { -// return &ExplicitDeleteTask{ -// e: e, -// owner: owner, -// priority: priority, -// } -// } - -// func (edt *ExplicitDeleteTask) Run(ctx context.Context, submit func(pool.Task) error) error { - -// if !edt.e.HoldsLeafvariants() { -// return nil -// } -// le := edt.e.GetLeafVariantEntries().GetByOwner(edt.owner) -// if le != nil { -// le.MarkExpliciteDelete() -// } else { -// le = edt.e.GetLeafVariantEntries().AddExplicitDeleteEntry(edt.owner, edt.priority) -// } - -// for _, c := range edt.e.GetChilds(DescendMethodAll) { -// deleteTask := NewExplicitDeleteTask(c, edt.owner, edt.priority) -// err := submit(deleteTask) -// if err != nil { -// return err -// } -// } -// return nil - -// } - -// func ExplicitDelete(ctx context.Context, e Entry, dps *DeletePathSet, pool pool.VirtualPoolI) error { -// for dp := range dps.Items() { -// for pi := range dp.PathItems() { -// edp, err := e.NavigateSdcpbPath(ctx, pi) -// if err != nil { -// return err -// } - -// deleteTask := NewExplicitDeleteTask(edp, dp.GetOwner(), dp.GetPrio()) - -// err = pool.Submit(deleteTask) -// if err != nil { -// return err -// } -// } -// } - -// // close pool for additional external submission -// pool.CloseForSubmit() -// // wait for the pool to run dry -// pool.Wait() -// return nil -// } diff --git a/pkg/tree/xml.go b/pkg/tree/xml.go index f71b4804..e568a8b8 100644 --- a/pkg/tree/xml.go +++ b/pkg/tree/xml.go @@ -7,6 +7,7 @@ import ( "sort" "github.com/beevik/etree" + "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) @@ -42,7 +43,7 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp // if the entry remains so exist, we need to add it to the xml doc overallDoAdd := false - childs := s.GetChilds(DescendMethodActiveChilds) + childs := s.GetChilds(types.DescendMethodActiveChilds) keys := make([]string, 0, len(childs)) for k := range childs { diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index f85fff42..8084a4ff 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -546,35 +546,11 @@ func TestToXMLTable(t *testing.T) { } owner := "owner1" - // var runningCacheUpds []*types.Update - // if tt.runningConfig != nil { - // runningSdcpbUpds, err := tt.runningConfig(context.Background(), utils.NewConverter(scb)) - // if err != nil { - // t.Error(err) - // } - // runningCacheUpds, err = utils.SdcpbUpdatesToCacheUpdates(runningSdcpbUpds, RunningIntentName, RunningValuesPrio) - // if err != nil { - // t.Error(err) - // } - // } - - // var intendedCacheUpds []*types.Update - // if tt.existingConfig != nil { - // intendedSdcpbUpds, err := tt.existingConfig(context.Background(), utils.NewConverter(scb)) - // if err != nil { - // t.Error(err) - // } - // intendedCacheUpds, err = utils.SdcpbUpdatesToCacheUpdates(intendedSdcpbUpds, owner, 5) - // if err != nil { - // t.Error(err) - // } - // } - ctx := context.Background() converter := utils.NewConverter(scb) - tc := NewTreeContext(scb, owner) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -593,11 +569,10 @@ func TestToXMLTable(t *testing.T) { fmt.Println(root.String()) if tt.newConfig != nil { - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 53d78944..8d170367 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -2,17 +2,13 @@ package testhelper import ( "context" - "encoding/json" "fmt" "slices" "strings" "testing" "github.com/google/go-cmp/cmp" - "github.com/openconfig/ygot/ygot" "github.com/sdcio/data-server/mocks/mockschemaclientbound" - "github.com/sdcio/data-server/pkg/tree/importer" - jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "go.uber.org/mock/gomock" @@ -137,30 +133,13 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem return mockscb, nil } -type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error -} - -func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, flags *types.UpdateInsertFlags) error { - jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ - Format: ygot.RFC7951, - SkipValidation: true, - }) - if err != nil { - return err - } - - var jsonConfAny any - err = json.Unmarshal([]byte(jconfStr), &jsonConfAny) - if err != nil { - return err - } - - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner, prio, flags) - if err != nil { - return err - } - return nil +type ImportStatsInterface interface { + Changed() bool + GetNewCount() int64 + GetUpdatedCount() int64 + IncrementNew() + IncrementUpdated() + String() string } // SplitStringSortDiff split the two strings a and b on sep, sort alphabetical and return the diff