From 92fab4f04d665da02508e787967c4ee6f1f9a96e Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 15 Jan 2026 10:51:36 +0100 Subject: [PATCH 01/17] add non-revertive flag to transaction intents --- pkg/datastore/datastore_rpc.go | 2 +- pkg/datastore/deviations.go | 2 +- pkg/datastore/intent_rpc.go | 2 +- pkg/datastore/sync.go | 2 +- pkg/datastore/transaction_rpc.go | 17 +++++----- pkg/datastore/tree_operation_test.go | 3 +- .../tree_operation_validation_test.go | 3 +- pkg/datastore/types/transaction_intent.go | 16 +++++----- .../proto/proto_tree_importer_test.go | 4 +-- .../importer/xml/xml_tree_importer_test.go | 2 +- pkg/tree/processor_blame_config_test.go | 10 +++--- pkg/tree/root_entry.go | 6 ++-- pkg/tree/root_entry_test.go | 8 ++--- pkg/tree/sharedEntryAttributes_test.go | 30 +++++++++--------- pkg/tree/tree_context.go | 31 +++++++++++++++---- pkg/tree/validation_entry_leafref_test.go | 2 +- pkg/tree/validation_range_test.go | 6 ++-- pkg/tree/visitor_explicit_delete_test.go | 12 +++---- pkg/utils/testhelper/utils.go | 6 ++-- 19 files changed, 90 insertions(+), 74 deletions(-) diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 1e030e09..c57b9125 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -239,7 +239,7 @@ func (d *Datastore) BlameConfig(ctx context.Context, includeDefaults bool) (*sdc return nil, err } // load all intents - _, err = d.LoadAllButRunningIntents(ctx, root, true) + _, err = d.LoadAllButRunningIntents(ctx, root) if err != nil { return nil, err } diff --git a/pkg/datastore/deviations.go b/pkg/datastore/deviations.go index cf8e60bf..7681e3c9 100644 --- a/pkg/datastore/deviations.go +++ b/pkg/datastore/deviations.go @@ -169,7 +169,7 @@ func (d *Datastore) calculateDeviations(ctx context.Context) (<-chan *treetypes. return nil, err } - addedIntentNames, err := d.LoadAllButRunningIntents(ctx, deviationTree, true) + addedIntentNames, err := d.LoadAllButRunningIntents(ctx, deviationTree) if err != nil { return nil, err } diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 6ce84237..80df1ec2 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -77,7 +77,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, tp.GetIntentName(), tp.GetPriority(), types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, protoImporter, tp.GetIntentName(), tp.GetPriority(), tp.GetNonRevertive(), types.NewUpdateInsertFlags()) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index bb43a051..90e086cc 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -26,7 +26,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } if importer != nil { - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, tree.RunningIntentName, tree.RunningValuesPrio, treetypes.NewUpdateInsertFlags()) + err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, tree.RunningIntentName, tree.RunningValuesPrio, false, treetypes.NewUpdateInsertFlags()) if err != nil { return err } diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 4cb07e20..97b3c867 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -41,8 +41,8 @@ func (d *Datastore) SdcpbTransactionIntentToInternalTI(ctx context.Context, req if req.GetOrphan() { ti.SetDeleteOnlyIntendedFlag() } - if req.GetDeviation() { - ti.SetDeviation() + if req.GetNonRevertive() { + ti.SetNonRevertive() } if req.GetDeleteIgnoreNoExist() { ti.SetDeleteIgnoreNonExisting() @@ -86,7 +86,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), tree.RunningIntentName, tree.RunningValuesPrio, treetypes.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), tree.RunningIntentName, tree.RunningValuesPrio, false, treetypes.NewUpdateInsertFlags()) if err != nil { return nil, err } @@ -147,7 +147,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa return warnings, nil } -func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry, excludeDeviations bool) ([]string, error) { +func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry) ([]string, error) { log := logf.FromContext(ctx) intentNames := []string{} @@ -174,15 +174,12 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo IntentChan = nil break selectLoop } - if excludeDeviations && intent.Deviation { - continue - } log.V(logf.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) log.V(logf.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.FormatProtoJSON(intent)) intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, intent.GetIntentName(), intent.GetPriority(), treetypes.NewUpdateInsertFlags()) + err := root.ImportConfig(ctx, nil, protoLoader, intent.GetIntentName(), intent.GetPriority(), intent.GetNonRevertive(), treetypes.NewUpdateInsertFlags()) if err != nil { return nil, err } @@ -204,7 +201,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ return nil, err } - _, err = d.LoadAllButRunningIntents(ctx, root, false) + _, err = d.LoadAllButRunningIntents(ctx, root) if err != nil { return nil, err } @@ -354,7 +351,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ delSl := deletesOwner.ToXPathSlice() log.V(logf.VTrace).Info("deletes owner", "deletes-owner", delSl) - protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority(), intent.Deviation()) + protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority(), intent.NonRevertive()) switch { case errors.Is(err, tree.ErrorIntentNotPresent): err = d.cacheClient.IntentDelete(ctx, intent.GetName(), intent.GetDeleteIgnoreNonExisting()) diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 0105746c..c6aa058d 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -75,6 +75,7 @@ func TestDatastore_populateTree(t *testing.T) { intendedStoreUpdates []*types.PathAndUpdate runningStoreUpdates []*types.PathAndUpdate NotOnlyNewOrUpdated bool // it negated when used in the call, usually we want it to be true + nonRevertive bool }{ { name: "DoubleKey - Delete Single item", @@ -1518,7 +1519,7 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny), tt.intentName, tt.intentPrio, newFlag) + err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny), tt.intentName, tt.intentPrio, tt.nonRevertive, newFlag) if err != nil { t.Error(err) } diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index ed93d911..fc020302 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -59,6 +59,7 @@ func TestDatastore_validateTree(t *testing.T) { intendedStoreUpdates []*cache.Update NotOnlyNewOrUpdated bool // it negated when used in the call, usually we want it to be true expectedWarnings []string + nonRevertive bool }{ { @@ -197,7 +198,7 @@ func TestDatastore_validateTree(t *testing.T) { importer := json_importer.NewJsonTreeImporter(jsonConf) - err = root.ImportConfig(ctx, path, importer, tt.intentName, tt.intentPrio, flagsNew) + err = root.ImportConfig(ctx, path, importer, tt.intentName, tt.intentPrio, tt.nonRevertive, flagsNew) if err != nil { t.Error(err) } diff --git a/pkg/datastore/types/transaction_intent.go b/pkg/datastore/types/transaction_intent.go index 31a77ba9..db65f307 100644 --- a/pkg/datastore/types/transaction_intent.go +++ b/pkg/datastore/types/transaction_intent.go @@ -11,11 +11,9 @@ type TransactionIntent struct { updates []*treetypes.PathAndUpdate delete bool // onlyIntended, the orphan flag, delte only from intended store, but keep in device - onlyIntended bool - priority int32 - // deviation indicates that the intent is a tolerated deviation. - // it will be stored and used for change calculation but will be excluded when claculating actual deviations. - deviation bool + onlyIntended bool + priority int32 + nonRevertive bool deleteIgnoreNonExisting bool explicitDeletes *sdcpb.PathSet } @@ -57,12 +55,12 @@ func (ti *TransactionIntent) GetOnlyIntended() bool { return ti.onlyIntended } -func (ti *TransactionIntent) SetDeviation() { - ti.deviation = true +func (ti *TransactionIntent) SetNonRevertive() { + ti.nonRevertive = true } -func (ti *TransactionIntent) Deviation() bool { - return ti.deviation +func (ti *TransactionIntent) NonRevertive() bool { + return ti.nonRevertive } func (ti *TransactionIntent) SetDeleteIgnoreNonExisting() { diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index a762d33f..8e9a2e9a 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -127,7 +127,7 @@ func TestProtoTreeImporter(t *testing.T) { } jti := jimport.NewJsonTreeImporter(j) - err = root.ImportConfig(ctx, nil, jti, "owner1", 5, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, jti, "owner1", 5, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatal(err) } @@ -153,7 +153,7 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, protoIntent.GetIntentName(), protoIntent.GetPriority(), types.NewUpdateInsertFlags()) + err = rootNew.ImportConfig(ctx, nil, protoAdapter, protoIntent.GetIntentName(), protoIntent.GetPriority(), false, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index 4ace5325..d22720ec 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -92,7 +92,7 @@ func TestXmlTreeImporter(t *testing.T) { t.Fatal(err) } - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element), "owner1", 5, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element), "owner1", 5, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index ea7ca80c..45a941a5 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -46,7 +46,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -111,13 +111,13 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } conf2 := config2() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -132,7 +132,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 8eea8a8f..53341d59 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -88,7 +88,7 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error { +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { r.treeContext.SetActualOwner(intentName) e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) @@ -184,7 +184,7 @@ func (r *RootEntry) GetDeviations(ctx context.Context, ch chan<- *types.Deviatio r.sharedEntryAttributes.GetDeviations(ctx, ch, true) } -func (r *RootEntry) TreeExport(owner string, priority int32, deviation bool) (*tree_persist.Intent, error) { +func (r *RootEntry) TreeExport(owner string, priority int32, nonRevertive bool) (*tree_persist.Intent, error) { treeExport, err := r.sharedEntryAttributes.TreeExport(owner) if err != nil { return nil, err @@ -202,7 +202,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32, deviation bool) (*t IntentName: owner, Root: rootExportEntry, Priority: priority, - Deviation: deviation, + NonRevertive: nonRevertive, ExplicitDeletes: explicitDeletes, }, nil } diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index ce4d0d05..af432937 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -399,7 +399,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } @@ -566,11 +566,11 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) if err != nil { t.Fatal(err) } @@ -587,7 +587,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index c47c8601..ce550633 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -146,7 +146,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Error(err) } @@ -208,11 +208,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -246,11 +246,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -344,7 +344,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -459,7 +459,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -474,7 +474,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } @@ -654,7 +654,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Fatal(err) } @@ -776,7 +776,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Fatal(err) } @@ -917,7 +917,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -974,7 +974,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -1130,7 +1130,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), owner1, owner1Prio, flagsExisting) + err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) return @@ -1282,7 +1282,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Fatal(err) } @@ -1451,7 +1451,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index 8dc1d19c..0acc2230 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -7,23 +7,42 @@ import ( ) type TreeContext struct { - root Entry // the trees root element - schemaClient schemaClient.SchemaClientBound - actualOwner string + root Entry // the trees root element + schemaClient schemaClient.SchemaClientBound + actualOwner string + nonRevertiveInfo map[string]bool } func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *TreeContext { return &TreeContext{ - schemaClient: sc, - actualOwner: actualOwner, + schemaClient: sc, + actualOwner: actualOwner, + nonRevertiveInfo: map[string]bool{}, } } // deepCopy root is required to be set manually func (t *TreeContext) deepCopy() *TreeContext { - return &TreeContext{ + tc := &TreeContext{ schemaClient: t.schemaClient, } + + // deepcopy nonRevertiveInfo + m := map[string]bool{} + for k, v := range t.nonRevertiveInfo { + m[k] = v + } + tc.nonRevertiveInfo = m + return tc +} + +func (t *TreeContext) AddNonRevertiveInfo(intent string, nonRevertive bool) { + t.nonRevertiveInfo[intent] = nonRevertive +} + +// IsNonRevertiveIntent returns the non-revertive flag per intent. False is also returned the intent does not exist. +func (t *TreeContext) IsNonRevertiveIntent(intent string) bool { + return t.nonRevertiveInfo[intent] } func (t *TreeContext) SetRoot(e Entry) error { diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 944aff6d..7d9b4722 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -236,7 +236,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index bed8495a..19b8c95f 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -67,7 +67,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } @@ -182,7 +182,7 @@ func TestValidate_RangesSigned(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } @@ -316,7 +316,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } diff --git a/pkg/tree/visitor_explicit_delete_test.go b/pkg/tree/visitor_explicit_delete_test.go index 172b44cb..b66d3073 100644 --- a/pkg/tree/visitor_explicit_delete_test.go +++ b/pkg/tree/visitor_explicit_delete_test.go @@ -53,7 +53,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsNew) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) if err != nil { t.Error(err) } @@ -81,12 +81,12 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -133,12 +133,12 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, flagsExisting) + err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -148,7 +148,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { Name: ygot.String("ethernet-1/1"), Description: ygot.String("mydesc"), }, - }}, root, owner2, owner2Prio, flagsNew) + }}, root, owner2, owner2Prio, false, flagsNew) return root }, diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 53d78944..a9fa5cec 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -138,10 +138,10 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem } type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error } -func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, flags *types.UpdateInsertFlags) error { +func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, SkipValidation: true, @@ -156,7 +156,7 @@ func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root Root return err } - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner, prio, flags) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner, prio, nonRevertive, flags) if err != nil { return err } From 06e88e887dfd0d40cd23920a1c979cb00f09092c Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 15 Jan 2026 10:55:33 +0100 Subject: [PATCH 02/17] tmp adjust go.mod --- go.mod | 20 +++++++++++--------- go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 8a88acb5..597b9c22 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.24.0 replace github.com/openconfig/goyang v1.6.0 => github.com/sdcio/goyang v1.6.2-2 +replace github.com/sdcio/sdc-protos => /home/mava/projects/sdc-protos + require ( github.com/AlekSi/pointer v1.2.0 github.com/beevik/etree v1.6.0 @@ -29,8 +31,8 @@ require ( github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 go.uber.org/mock v0.6.0 - google.golang.org/grpc v1.77.0 - google.golang.org/protobuf v1.36.10 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/controller-runtime v0.20.4 ) @@ -89,16 +91,16 @@ require ( go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/crypto v0.43.0 // indirect + golang.org/x/crypto v0.44.0 // indirect golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect - golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.32.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/term v0.36.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.8.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fabbbc23..efc8a2b1 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,6 @@ github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= github.com/sdcio/logger v0.0.3/go.mod h1:yWaOxK/G6vszjg8tKZiMqiEjlZouHsjFME4zSk+SAEA= github.com/sdcio/schema-server v0.0.33 h1:RTeQMIchynAPSQaf61CZBgYHCCpsyDLEAWJn+ZKImIo= github.com/sdcio/schema-server v0.0.33/go.mod h1:q8leN1KhRNTnnqf6yxvkDk5tFl6DAsHcl81usVgYpoI= -github.com/sdcio/sdc-protos v0.0.47 h1:1SD8Ifx7MRc6fqjuJIM1wQ2NGEwjYj+QWgHNgENJiT4= -github.com/sdcio/sdc-protos v0.0.47/go.mod h1:cVCYbMq+tu7EKXyJKFq3p2bsCwjvXbxwsyOoMGiQddw= github.com/sdcio/yang-parser v0.0.12 h1:RSSeqfAOIsJx5Lno5u4/ezyOmQYUduQ22rBfU/mtpJ4= github.com/sdcio/yang-parser v0.0.12/go.mod h1:CBqn3Miq85qmFVGHxHXHLluXkaIOsTzV06IM4DW6+D4= github.com/sirikothe/gotextfsm v1.0.1-0.20200816110946-6aa2cfd355e4 h1:FHUL2HofYJuslFOQdy/JjjP36zxqIpd/dcoiwLMIs7k= @@ -244,8 +242,8 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= @@ -263,8 +261,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= -golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= @@ -273,8 +271,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -284,14 +282,14 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -303,8 +301,8 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -318,17 +316,17 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= -google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 64a988af2d1460cfbc25e13cd2aa45b04d05ce35 Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 19 Jan 2026 13:37:49 +0100 Subject: [PATCH 03/17] Make the syncing of running not blindly overwrite but transfor the sync tree --- pkg/datastore/sync.go | 52 +++- pkg/datastore/sync_test.go | 379 +++++++++++++++++++++++++ pkg/tree/entry.go | 3 + pkg/tree/leaf_variants.go | 37 +++ pkg/tree/processor_remove_deleted.go | 102 +++++++ pkg/tree/processor_reset_flags.go | 83 ++++++ pkg/tree/processor_reset_flags_test.go | 124 ++++++++ 7 files changed, 778 insertions(+), 2 deletions(-) create mode 100644 pkg/datastore/sync_test.go create mode 100644 pkg/tree/processor_remove_deleted.go create mode 100644 pkg/tree/processor_reset_flags.go create mode 100644 pkg/tree/processor_reset_flags_test.go diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 90e086cc..91946a50 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -2,7 +2,9 @@ package datastore import ( "context" + "fmt" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" treetypes "github.com/sdcio/data-server/pkg/tree/types" @@ -17,14 +19,32 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i d.syncTreeMutex.Lock() defer d.syncTreeMutex.Unlock() + + // create a virtual task pool for delete operations + deleteMarkerPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) for _, delete := range deletes { - err := d.syncTree.DeleteBranch(ctx, delete, tree.RunningIntentName) + // navigate to delete path + deleteRoot, err := d.syncTree.NavigateSdcpbPath(ctx, delete) if err != nil { - log.Error(err, "failed deleting path from datastore sync tree", "severity", "WARN", "path", delete.ToXPath(false)) + log.Error(err, "failed navigating to delete path", "path", delete.ToXPath(false)) continue } + // apply delete marker, setting owner delete flag on running intent + err = tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tree.RunningIntentName, false)).Run(deleteRoot, deleteMarkerPool) + if err != nil { + log.Error(err, "failed applying delete to path", "path", delete.ToXPath(false)) + continue + } + } + // close the delete marker pool for submission + deleteMarkerPool.CloseForSubmit() + deleteMarkerPool.Wait() + err := deleteMarkerPool.FirstError() + if err != nil { + return err } + // import new config if provided if importer != nil { err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, tree.RunningIntentName, tree.RunningValuesPrio, false, treetypes.NewUpdateInsertFlags()) if err != nil { @@ -32,6 +52,34 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } } + // create a virtual task pool for remove deleted operations + removeDeletedPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + + // run remove deleted processor to clean up entries marked as deleted by owner + delProcessorParams := tree.NewRemoveDeletedProcessorParameters(tree.RunningIntentName) + err = tree.NewRemoveDeletedProcessor(delProcessorParams).Run(d.syncTree.GetRoot(), removeDeletedPool) + if err != nil { + return err + } + + // close the remove deleted pool for submission + removeDeletedPool.CloseForSubmit() + removeDeletedPool.Wait() + err = removeDeletedPool.FirstError() + if err != nil { + return err + } + + // delete entries that have zero-length leaf variant entries after remove deleted processing + for _, e := range delProcessorParams.GetZeroLengthLeafVariantEntries() { + fmt.Println("entry has zero-length leaf variant entries after remove deleted", "entry", e.SdcpbPath().ToXPath(false)) + + err := e.GetParent().DeleteBranch(ctx, &sdcpb.Path{Elem: []*sdcpb.PathElem{sdcpb.NewPathElem(e.PathName(), nil)}}, tree.RunningIntentName) + if err != nil { + return err + } + } + // conditional trace logging if log := log.V(logger.VTrace); log.Enabled() { treeExport, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go new file mode 100644 index 00000000..ddeaa91c --- /dev/null +++ b/pkg/datastore/sync_test.go @@ -0,0 +1,379 @@ +package datastore + +import ( + "context" + "encoding/json" + "fmt" + "runtime" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/openconfig/ygot/ygot" + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree" + "github.com/sdcio/data-server/pkg/tree/importer" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +// TestApplyToRunning tests the ApplyToRunning method of the Datastore struct. +func TestApplyToRunning(t *testing.T) { + // Setup + ctx := context.Background() + + // Define test cases + tests := []struct { + name string + deletes []*sdcpb.Path + importer func() importer.ImportConfigAdapter + syncTree func() *tree.RootEntry + result func() any + wantErr bool + }{ + { + name: "delete entire interface (e1-1 existed now e1-2 added)", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTree: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, tree.RunningIntentName) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importer: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/2": { + Name: ygot.String("ethernet-1/2"), + Description: ygot.String("1/2 description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v) + }, + result: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/2": { + Name: ygot.String("ethernet-1/2"), + Description: ygot.String("1/2 description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + }, + { + name: "delete description of existing interface", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTree: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, tree.RunningIntentName) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importer: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v) + }, + result: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + }, + { + name: "change description of existing interface", + deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, + syncTree: func() *tree.RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, tree.RunningIntentName) + + root, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + return root + + }, + importer: func() importer.ImportConfigAdapter { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my other description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + + return jsonImporter.NewJsonTreeImporter(v) + }, + result: func() any { + d := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my other description"), + }, + }, + } + confStr, err := ygot.EmitJSON(d, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal test config: %v", err) + } + + var v any + json.Unmarshal([]byte(confStr), &v) + return v + }, + wantErr: false, + }, + } + + // Run tests + for _, tt := range tests { + + fmt.Println("----" + tt.name) + + t.Run(tt.name, func(t *testing.T) { + syncTree := tt.syncTree() + + datastore := &Datastore{ + syncTreeMutex: &sync.RWMutex{}, + syncTree: syncTree, + taskPool: pool.NewSharedTaskPool(ctx, runtime.NumCPU()), + } + err := datastore.ApplyToRunning(ctx, tt.deletes, tt.importer()) + if (err != nil) != tt.wantErr { + t.Errorf("ApplyToRunning() error = %v, wantErr %v", err, tt.wantErr) + } + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := tree.NewTreeContext(scb, tree.RunningIntentName) + + resultRoot, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + d := tt.result() + + err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + if err != nil { + t.Fatalf("failed to import test config: %v", err) + } + + err = resultRoot.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + + fmt.Println(syncTree.String()) + + vpool := datastore.taskPool.NewVirtualPool(pool.VirtualFailFast, runtime.NumCPU()) + + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), vpool) + if err != nil { + t.Fatalf("failed to reset flags: %v", err) + } + + vpool.CloseForSubmit() + vpool.Wait() + err = vpool.FirstError() + if err != nil { + t.Fatalf("failed to run reset flags processor: %v", err) + } + + fmt.Println("Adjusted flags count:", resetFlagsProcessorParams.GetAdjustedFlagsCount()) + + if diff := cmp.Diff(resultRoot.String(), syncTree.String()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index dd6fef22..0238382f 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -163,12 +163,15 @@ type LeafVariantEntry interface { type LeafVariantEntries interface { MarkOwnerForDeletion(owner string, onlyIntended bool) *LeafEntry + ResetFlags(deleteFlag bool, newFlag bool, updatedFlag bool) int GetHighestPrecedence(onlyNewOrUpdated bool, includeDefaults bool, includeExplicitDeletes bool) *LeafEntry GetRunning() *LeafEntry DeleteByOwner(owner string) *LeafEntry AddExplicitDeleteEntry(owner string, priority int32) *LeafEntry GetByOwner(owner string) *LeafEntry + RemoveDeletedByOwner(owner string) *LeafEntry Add(l *LeafEntry) + Length() int } type DescendMethod int diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index e7d0d7ef..9bf01699 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -89,6 +89,20 @@ func (lv *LeafVariants) canDeleteBranch(keepDefault bool) bool { return true } +// RemoveDeletedByOwner removes and returns the LeafEntry owned by the given owner if it is marked for deletion. +func (lv *LeafVariants) RemoveDeletedByOwner(owner string) *LeafEntry { + lv.lesMutex.Lock() + defer lv.lesMutex.Unlock() + for i, l := range lv.les { + if l.Owner() == owner && l.GetDeleteFlag() { + // Remove element from slice + lv.les = append(lv.les[:i], lv.les[i+1:]...) + return l + } + } + return nil +} + // canDelete returns true if leafValues exist that are not owned by default or running that do not have the DeleteFlag set [or if delete is set, also the DeleteOnlyIntendedFlag set] func (lv *LeafVariants) canDelete() bool { lv.lesMutex.RLock() @@ -367,6 +381,29 @@ func (lv *LeafVariants) MarkOwnerForDeletion(owner string, onlyIntended bool) *L return nil } +func (lv *LeafVariants) ResetFlags(deleteFlag bool, newFlag bool, updatedFlag bool) int { + lv.lesMutex.Lock() + defer lv.lesMutex.Unlock() + count := 0 + + for _, le := range lv.les { + if deleteFlag && le.Delete { + le.Delete = false + le.DeleteOnlyIntended = false + count++ + } + if updatedFlag && le.IsUpdated { + le.IsUpdated = false + count++ + } + if newFlag && le.IsNew { + le.IsNew = false + count++ + } + } + return count +} + func (lv *LeafVariants) DeleteByOwner(owner string) *LeafEntry { lv.lesMutex.Lock() defer lv.lesMutex.Unlock() diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go new file mode 100644 index 00000000..cce338a3 --- /dev/null +++ b/pkg/tree/processor_remove_deleted.go @@ -0,0 +1,102 @@ +package tree + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/sdcio/data-server/pkg/pool" +) + +type RemoveDeletedProcessor struct { + config *RemoveDeletedProcessorParameters +} + +func NewRemoveDeletedProcessor(c *RemoveDeletedProcessorParameters) *RemoveDeletedProcessor { + return &RemoveDeletedProcessor{ + config: c, + } +} + +type RemoveDeletedProcessorParameters struct { + owner string + deleteStatsCount atomic.Int64 + zeroLeafEntryElements []Entry + zeroLeafEntryElementsLock sync.Mutex +} + +func NewRemoveDeletedProcessorParameters(owner string) *RemoveDeletedProcessorParameters { + return &RemoveDeletedProcessorParameters{ + owner: owner, + deleteStatsCount: atomic.Int64{}, + zeroLeafEntryElements: []Entry{}, + zeroLeafEntryElementsLock: sync.Mutex{}, + } +} + +func (r *RemoveDeletedProcessorParameters) GetDeleteStatsCount() int64 { + return r.deleteStatsCount.Load() +} + +// GetZeroLengthLeafVariantEntries returns the entries that have zero-length leaf variant entries after removal +func (r *RemoveDeletedProcessorParameters) GetZeroLengthLeafVariantEntries() []Entry { + return r.zeroLeafEntryElements +} + +func (o *RemoveDeletedProcessor) Run(e Entry, pool pool.VirtualPoolI) error { + err := pool.Submit(newRemoveDeletedTask(o.config, e, false)) + if err != nil { + return err + } + // close pool for additional external submission + pool.CloseForSubmit() + // wait for the pool to run dry + pool.Wait() + + return pool.FirstError() +} + +type removeDeletedTask struct { + config *RemoveDeletedProcessorParameters + e Entry + keepDefaults bool +} + +func newRemoveDeletedTask(c *RemoveDeletedProcessorParameters, e Entry, keepDefaults bool) *removeDeletedTask { + return &removeDeletedTask{ + config: c, + e: e, + keepDefaults: keepDefaults, + } +} + +func (t *removeDeletedTask) Run(ctx context.Context, submit func(pool.Task) error) error { + if ctx.Err() != nil { + return ctx.Err() + } + + res := t.e.GetLeafVariantEntries().RemoveDeletedByOwner(t.config.owner) + if res != nil { + // increment the delete stats count + t.config.deleteStatsCount.Add(1) + } + if t.e.canDeleteBranch(t.keepDefaults) { + func() { + t.config.zeroLeafEntryElementsLock.Lock() + defer t.config.zeroLeafEntryElementsLock.Unlock() + t.config.zeroLeafEntryElements = append(t.config.zeroLeafEntryElements, t.e) + }() + return nil + } + + // process childs + for _, c := range t.e.GetChilds(DescendMethodAll) { + childTask := newRemoveDeletedTask(t.config, c, t.e.GetSchema().GetContainer() == nil) + err := submit(childTask) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go new file mode 100644 index 00000000..5275199e --- /dev/null +++ b/pkg/tree/processor_reset_flags.go @@ -0,0 +1,83 @@ +package tree + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/sdcio/data-server/pkg/pool" +) + +// ResetFlagsProcessor resets the flags on leaf variant entries +type ResetFlagsProcessor struct { + config *ResetFlagsProcessorParameters +} + +func NewResetFlagsProcessor(c *ResetFlagsProcessorParameters) *ResetFlagsProcessor { + return &ResetFlagsProcessor{ + config: c, + } +} + +type ResetFlagsProcessorParameters struct { + deleteFlag, newFlag, updateFlag bool + adjustedFlagsCount atomic.Int64 +} + +func NewResetFlagsProcessorParameters(deleteFlag, newFlag, updateFlag bool) *ResetFlagsProcessorParameters { + return &ResetFlagsProcessorParameters{ + deleteFlag: deleteFlag, + newFlag: newFlag, + updateFlag: updateFlag, + adjustedFlagsCount: atomic.Int64{}, + } +} + +// GetAdjustedFlagsCount returns the number of flags that were adjusted +func (r *ResetFlagsProcessorParameters) GetAdjustedFlagsCount() int64 { + return r.adjustedFlagsCount.Load() +} + +func (o *ResetFlagsProcessor) Run(e Entry, pool pool.VirtualPoolI) error { + if e == nil { + return fmt.Errorf("entry cannot be nil") + } + err := pool.Submit(newResetFlagsTask(o.config, e)) + if err != nil { + return err + } + // close pool for additional external submission + pool.CloseForSubmit() + // wait for the pool to run dry + pool.Wait() + + return pool.FirstError() +} + +type resetFlagsTask struct { + config *ResetFlagsProcessorParameters + e Entry +} + +func newResetFlagsTask(config *ResetFlagsProcessorParameters, e Entry) *resetFlagsTask { + return &resetFlagsTask{ + config: config, + e: e, + } +} + +func (t *resetFlagsTask) Run(ctx context.Context, submit func(pool.Task) error) error { + // reset flags as per config + count := t.e.GetLeafVariantEntries().ResetFlags(t.config.deleteFlag, t.config.newFlag, t.config.updateFlag) + t.config.adjustedFlagsCount.Add(int64(count)) + + // process childs + for _, c := range t.e.GetChilds(DescendMethodAll) { + err := submit(newResetFlagsTask(t.config, c)) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go new file mode 100644 index 00000000..876e779d --- /dev/null +++ b/pkg/tree/processor_reset_flags_test.go @@ -0,0 +1,124 @@ +package tree + +import ( + "context" + "fmt" + "runtime" + "testing" + + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResetFlagsProcessorRun tests the Run method of ResetFlagsProcessor +func TestResetFlagsProcessorRun(t *testing.T) { + tests := []struct { + name string + deleteFlag bool + newFlag bool + updateFlag bool + wantErr bool + tree func() *RootEntry + adjustCount int64 + }{ + { + name: "successful reset with all flags", + deleteFlag: true, + newFlag: true, + updateFlag: true, + wantErr: false, + adjustCount: 4, + tree: func() *RootEntry { + + ctx := context.Background() + + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + tc := NewTreeContext(scb, RunningIntentName) + + root, err := NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create new tree root: %v", err) + } + + _, err = root.AddUpdateRecursive(ctx, + &sdcpb.Path{Elem: []*sdcpb.PathElem{ + sdcpb.NewPathElem("interface", map[string]string{"name": "ethernet-1/1"}), + sdcpb.NewPathElem("description", nil)}, + }, + types.NewUpdate(nil, testhelper.GetStringTvProto("desc 1/1"), RunningValuesPrio, RunningIntentName, 0), + flagsNew, + ) + if err != nil { + t.Fatalf("failed to add update recursive: %v", err) + } + + _, err = root.AddUpdateRecursive(ctx, + &sdcpb.Path{Elem: []*sdcpb.PathElem{ + sdcpb.NewPathElem("interface", map[string]string{"name": "ethernet-1/2"}), + sdcpb.NewPathElem("description", nil)}, + }, + types.NewUpdate(nil, testhelper.GetStringTvProto("desc 1/2"), RunningValuesPrio, RunningIntentName, 0), + types.NewUpdateInsertFlags().SetDeleteFlag(), + ) + if err != nil { + t.Fatalf("failed to add update recursive: %v", err) + } + + err = root.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + + return root + + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + + // Create processor parameters + params := NewResetFlagsProcessorParameters(tt.deleteFlag, tt.newFlag, tt.updateFlag) + + // Create processor + processor := NewResetFlagsProcessor(params) + require.NotNil(t, processor) + + // Create a mock entry for testing + // Note: In a real test, you would use a properly initialized Entry + // This is a simplified version - you may need to adjust based on your actual Entry implementation + + // Create a virtual pool for testing + taskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + virtualPool := taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + + root := tt.tree() + + fmt.Println(root.String()) + + processorErr := processor.Run(root.GetRoot(), virtualPool) + if (processorErr != nil) != tt.wantErr { + t.Errorf("ResetFlagsProcessor.Run() error = %v, wantErr %v", processorErr, tt.wantErr) + return + } + + fmt.Println(root.String()) + + // For now, we'll test that the processor can be created and run method exists + // Full integration testing would require properly mocked Entry objects + assert.NotNil(t, processor) + assert.Equal(t, int64(tt.adjustCount), params.GetAdjustedFlagsCount()) + }) + } +} From eccf554520c581591c777e360c297511647c74ce Mon Sep 17 00:00:00 2001 From: steiler Date: Wed, 21 Jan 2026 09:13:25 +0100 Subject: [PATCH 04/17] adjust pool error handling --- pkg/datastore/datastore_rpc.go | 2 +- pkg/datastore/sync.go | 23 ++--- pkg/datastore/sync_test.go | 5 +- pkg/datastore/target/gnmi/stream.go | 2 +- pkg/datastore/transaction_rpc.go | 2 +- pkg/datastore/tree_operation_test.go | 2 +- pkg/dslog/logger.go | 5 - pkg/pool/virtual_pool.go | 64 +++--------- pkg/pool/virtual_pool_iface.go | 6 +- pkg/pool/virtual_pool_test.go | 40 +------- pkg/tree/entry_test.go | 6 +- pkg/tree/processor_blame_config.go | 32 ++++-- pkg/tree/processor_blame_config_test.go | 2 +- pkg/tree/processor_error_collection_test.go | 107 ++++++++++++++++++++ pkg/tree/processor_mark_owner_delete.go | 29 ++++-- pkg/tree/processor_remove_deleted.go | 29 ++++-- pkg/tree/processor_reset_flags.go | 32 +++--- pkg/tree/processor_reset_flags_test.go | 2 +- pkg/tree/processor_validate.go | 5 +- pkg/tree/sharedEntryAttributes_test.go | 2 +- pkg/tree/xml_test.go | 2 +- 21 files changed, 235 insertions(+), 164 deletions(-) delete mode 100644 pkg/dslog/logger.go create mode 100644 pkg/tree/processor_error_collection_test.go diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index c57b9125..bb457be2 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -244,7 +244,7 @@ func (d *Datastore) BlameConfig(ctx context.Context, includeDefaults bool) (*sdc return nil, err } - blamePool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + blamePool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) bcp := tree.NewBlameConfigProcessor(tree.NewBlameConfigProcessorConfig(includeDefaults)) bte, err := bcp.Run(ctx, root.GetRoot(), blamePool) diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 91946a50..93760ac8 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -2,7 +2,7 @@ package datastore import ( "context" - "fmt" + "errors" "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" @@ -21,7 +21,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i defer d.syncTreeMutex.Unlock() // create a virtual task pool for delete operations - deleteMarkerPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteMarkerPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) for _, delete := range deletes { // navigate to delete path deleteRoot, err := d.syncTree.NavigateSdcpbPath(ctx, delete) @@ -36,9 +36,9 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i continue } } - // close the delete marker pool for submission - deleteMarkerPool.CloseForSubmit() - deleteMarkerPool.Wait() + + // close the delete marker pool for submission and wait + deleteMarkerPool.CloseAndWait() err := deleteMarkerPool.FirstError() if err != nil { return err @@ -52,8 +52,8 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } } - // create a virtual task pool for remove deleted operations - removeDeletedPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + // create a virtual task pool for removeDeleted operations + removeDeletedPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) // run remove deleted processor to clean up entries marked as deleted by owner delProcessorParams := tree.NewRemoveDeletedProcessorParameters(tree.RunningIntentName) @@ -62,18 +62,15 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i return err } - // close the remove deleted pool for submission - removeDeletedPool.CloseForSubmit() - removeDeletedPool.Wait() - err = removeDeletedPool.FirstError() + // close the remove deleted pool for submission and wait + removeDeletedPool.CloseAndWait() + err = errors.Join(removeDeletedPool.Errors()...) if err != nil { return err } // delete entries that have zero-length leaf variant entries after remove deleted processing for _, e := range delProcessorParams.GetZeroLengthLeafVariantEntries() { - fmt.Println("entry has zero-length leaf variant entries after remove deleted", "entry", e.SdcpbPath().ToXPath(false)) - err := e.GetParent().DeleteBranch(ctx, &sdcpb.Path{Elem: []*sdcpb.PathElem{sdcpb.NewPathElem(e.PathName(), nil)}}, tree.RunningIntentName) if err != nil { return err diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index ddeaa91c..fe9701b2 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -354,7 +354,7 @@ func TestApplyToRunning(t *testing.T) { fmt.Println(syncTree.String()) - vpool := datastore.taskPool.NewVirtualPool(pool.VirtualFailFast, runtime.NumCPU()) + vpool := datastore.taskPool.NewVirtualPool(pool.VirtualFailFast) resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), vpool) @@ -362,8 +362,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatalf("failed to reset flags: %v", err) } - vpool.CloseForSubmit() - vpool.Wait() + vpool.CloseAndWait() err = vpool.FirstError() if err != nil { t.Fatalf("failed to run reset flags processor: %v", err) diff --git a/pkg/datastore/target/gnmi/stream.go b/pkg/datastore/target/gnmi/stream.go index 03da073a..fd4f12af 100644 --- a/pkg/datastore/target/gnmi/stream.go +++ b/pkg/datastore/target/gnmi/stream.go @@ -176,7 +176,7 @@ func (s *StreamSync) gnmiSubscribe(subReq *gnmi.SubscribeRequest, updChan chan<- respChan, errChan := s.target.Subscribe(s.ctx, subReq, s.config.Name) - taskPool := s.vpoolFactory.NewVirtualPool(pool.VirtualTolerant, 10) + taskPool := s.vpoolFactory.NewVirtualPool(pool.VirtualTolerant) defer taskPool.CloseForSubmit() taskParams := NewNotificationProcessorTaskParameters(updChan, s.schemaClient) diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 97b3c867..b6cc10d5 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -219,7 +219,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ oldIntentContent := lvs.ToPathAndUpdateSlice() - deleteVisitorPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(intent.GetName(), intent.GetOnlyIntended())) err := ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index c6aa058d..2ed45c7f 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1502,7 +1502,7 @@ func TestDatastore_populateTree(t *testing.T) { } sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tt.intentName, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) diff --git a/pkg/dslog/logger.go b/pkg/dslog/logger.go deleted file mode 100644 index c5adb9fc..00000000 --- a/pkg/dslog/logger.go +++ /dev/null @@ -1,5 +0,0 @@ -package dslog - -const ( - TraceLevel = -8 -) diff --git a/pkg/pool/virtual_pool.go b/pkg/pool/virtual_pool.go index 850f4124..759c8172 100644 --- a/pkg/pool/virtual_pool.go +++ b/pkg/pool/virtual_pool.go @@ -42,20 +42,14 @@ func (f TaskFunc) Run(ctx context.Context, submit func(Task) error) error { // --- ErrorCollector (per-virtual tolerant mode) --- // ErrorCollector collects errors for a virtual pool. -// It stores a snapshotable slice and provides a live channel for streaming. +// It stores a snapshotable slice of errors. type ErrorCollector struct { mu sync.Mutex errs []error - Ch chan error } -func newErrorCollector(buf int) *ErrorCollector { - if buf <= 0 { - buf = 1024 - } - return &ErrorCollector{ - Ch: make(chan error, buf), - } +func newErrorCollector() *ErrorCollector { + return &ErrorCollector{} } func (ec *ErrorCollector) add(err error) { @@ -65,12 +59,6 @@ func (ec *ErrorCollector) add(err error) { ec.mu.Lock() defer ec.mu.Unlock() ec.errs = append(ec.errs, err) - - select { - case ec.Ch <- err: - default: - // drop if full - } } // Errors returns a snapshot of collected errors. @@ -82,11 +70,6 @@ func (ec *ErrorCollector) Errors() []error { return out } -// close channel when done -func (ec *ErrorCollector) close() { - close(ec.Ch) -} - // --- Virtual pool system --- var ErrVirtualPoolClosed = errors.New("virtual pool closed for submit") @@ -135,13 +118,8 @@ func (s *SharedTaskPool) Wait() error { } // NewVirtualPool creates and registers a virtual pool on top of the shared pool. -// id is an arbitrary identifier (must be unique per SharedTaskPool). -// mode controls failure semantics. buf controls error channel buffer for tolerant mode. -func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI { - // ensure unique id in the shared pool's map. If the requested id is already - // registered, append a short random hex postfix so multiple callers can - // create virtual pools with the same base name without colliding. - +// mode controls failure semantics. +func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode) VirtualPoolI { v := &VirtualPool{ parent: s, mode: mode, @@ -151,7 +129,7 @@ func (s *SharedTaskPool) NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI done: make(chan struct{}), } if mode == VirtualTolerant { - v.ec = newErrorCollector(buf) + v.ec = newErrorCollector() } return v } @@ -176,8 +154,6 @@ type VirtualPool struct { firstErr atomic.Pointer[error] // per-virtual inflight counter (matches lifecycle of tasks submitted by this virtual) inflight int64 - // ensure collector channel closed only once - collectorOnce sync.Once // ensure done channel closed only once (for Wait) waitOnce sync.Once // done is closed when the virtual pool is closed for submit and inflight reaches zero @@ -262,11 +238,6 @@ func (v *VirtualPool) Submit(t Task) error { func (v *VirtualPool) decrementInflight() { if remaining := atomic.AddInt64(&v.inflight, -1); remaining == 0 && v.closed.Load() { - v.collectorOnce.Do(func() { - if v.ec != nil { - v.ec.close() - } - }) v.waitOnce.Do(func() { close(v.done) }) @@ -302,14 +273,8 @@ func (v *VirtualPool) submitInternal(t Task) error { // when all virtual pools are done (call SharedTaskPool.CloseForSubmit()). func (v *VirtualPool) CloseForSubmit() { v.closed.Store(true) - // if nothing inflight, close collector now + // if nothing inflight, signal Wait() callers that virtual is drained if atomic.LoadInt64(&v.inflight) == 0 { - v.collectorOnce.Do(func() { - if v.ec != nil { - v.ec.close() - } - }) - // signal Wait() callers that virtual is drained v.waitOnce.Do(func() { close(v.done) }) @@ -323,6 +288,13 @@ func (v *VirtualPool) Wait() { <-v.done } +// CloseAndWait is a convenience method that closes the virtual pool for submission +// and waits for all inflight tasks to complete. +func (v *VirtualPool) CloseAndWait() { + v.CloseForSubmit() + v.Wait() +} + // isFailed returns true if this virtual pool encountered a fail-fast error. func (v *VirtualPool) isFailed() bool { if p := v.firstErr.Load(); p != nil && *p != nil { @@ -353,11 +325,3 @@ func (v *VirtualPool) Errors() []error { } return v.ec.Errors() } - -// ErrorChan returns the live channel of errors for tolerant mode, or nil for fail-fast mode. -func (v *VirtualPool) ErrorChan() <-chan error { - if v.ec == nil { - return nil - } - return v.ec.Ch -} diff --git a/pkg/pool/virtual_pool_iface.go b/pkg/pool/virtual_pool_iface.go index 396a34c2..1b2f9bb2 100644 --- a/pkg/pool/virtual_pool_iface.go +++ b/pkg/pool/virtual_pool_iface.go @@ -12,17 +12,17 @@ type VirtualPoolI interface { CloseForSubmit() // Wait blocks until the virtual has been closed for submit and all inflight tasks have completed. Wait() + // CloseAndWait is a convenience method that closes for submission and waits for drain. + CloseAndWait() // FirstError returns the first encountered error for fail-fast virtual pools, or nil. FirstError() error // Errors returns a snapshot of collected errors for tolerant virtual pools. Errors() []error - // ErrorChan returns the live channel of errors for tolerant mode, or nil for fail-fast mode. - ErrorChan() <-chan error } // Ensure VirtualPool implements the interface. var _ VirtualPoolI = (*VirtualPool)(nil) type VirtualPoolFactory interface { - NewVirtualPool(mode VirtualMode, buf int) VirtualPoolI + NewVirtualPool(mode VirtualMode) VirtualPoolI } diff --git a/pkg/pool/virtual_pool_test.go b/pkg/pool/virtual_pool_test.go index 086abc2c..acedca3a 100644 --- a/pkg/pool/virtual_pool_test.go +++ b/pkg/pool/virtual_pool_test.go @@ -51,8 +51,8 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { sp := NewSharedTaskPool(ctx, 4) // create virtual pools - vt := sp.NewVirtualPool(VirtualTolerant, 16) - vf := sp.NewVirtualPool(VirtualFailFast, 0) + vt := sp.NewVirtualPool(VirtualTolerant) + vf := sp.NewVirtualPool(VirtualFailFast) // submit tasks: tolerant pool will collect errors, fail pool will stop after first error var cntT int64 @@ -88,30 +88,10 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { // close shared pool for submit and wait sp.CloseForSubmit() - // drain tolerant error channel live - var seenErrs int32 - done := make(chan struct{}) - go func() { - for e := range vt.ErrorChan() { - if e != nil { - atomic.AddInt32(&seenErrs, 1) - } - } - close(done) - }() - if err := sp.Wait(); err != nil { t.Fatalf("shared wait error: %v", err) } - // tolerant collector channel is closed automatically when virtual is closed and inflight reaches zero - // wait for drain goroutine - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatalf("timeout waiting for tolerant drain") - } - // tolerant: expect 4 increments (one of the tasks had nested:1 which is now allowed even after CloseForSubmit) if got := atomic.LoadInt64(&cntT); got != 4 { t.Fatalf("tolerant counter expected 4 got %d", got) @@ -135,7 +115,7 @@ func TestVirtualPools_TolerantAndFailFast(t *testing.T) { func TestVirtualPool_Wait_Tolerant(t *testing.T) { ctx := context.Background() sp := NewSharedTaskPool(ctx, 2) - vt := sp.NewVirtualPool(VirtualTolerant, 4) + vt := sp.NewVirtualPool(VirtualTolerant) var cnt int64 // submit a few tasks @@ -157,16 +137,6 @@ func TestVirtualPool_Wait_Tolerant(t *testing.T) { // Wait on virtual specifically vt.Wait() - // ensure error channel closed (collector closed) - select { - case _, ok := <-vt.ErrorChan(): - if ok { - t.Fatalf("expected error channel to be closed") - } - default: - // channel may be drained; ensure snapshot is usable - } - if got := atomic.LoadInt64(&cnt); got != 2 { t.Fatalf("expected 2 increments, got %d", got) } @@ -176,7 +146,7 @@ func TestVirtualPool_Wait_Tolerant(t *testing.T) { func TestVirtualPool_Wait_FailFast(t *testing.T) { ctx := context.Background() sp := NewSharedTaskPool(ctx, 2) - vf := sp.NewVirtualPool(VirtualFailFast, 0) + vf := sp.NewVirtualPool(VirtualFailFast) var cnt int64 // submit a task that will fail and another that would be skipped when run @@ -207,7 +177,7 @@ func TestVirtualPool_Wait_FailFast(t *testing.T) { func TestVirtualPool_CancellationHang(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) sp := NewSharedTaskPool(ctx, 1) - vp := sp.NewVirtualPool(VirtualFailFast, 0) + vp := sp.NewVirtualPool(VirtualFailFast) var wg sync.WaitGroup wg.Add(2) diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index 70dc0185..beab79a7 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -567,7 +567,7 @@ func Test_Entry_Three(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) @@ -842,7 +842,7 @@ func Test_Entry_Four(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) @@ -1217,7 +1217,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) diff --git a/pkg/tree/processor_blame_config.go b/pkg/tree/processor_blame_config.go index 90063420..8c6e227e 100644 --- a/pkg/tree/processor_blame_config.go +++ b/pkg/tree/processor_blame_config.go @@ -2,6 +2,7 @@ package tree import ( "context" + "errors" "sync" "github.com/sdcio/data-server/pkg/pool" @@ -29,12 +30,16 @@ func NewBlameConfigProcessorConfig(includeDefaults bool) *BlameConfigProcessorCo } } +// Run processes the entry tree starting from e, building a blame tree showing which owner +// (intent) is responsible for each configuration value. The pool parameter should be +// VirtualFailFast to stop on first error. +// Returns the blame tree structure and any error encountered. func (p *BlameConfigProcessor) Run(ctx context.Context, e Entry, pool pool.VirtualPoolI) (*sdcpb.BlameTreeElement, error) { dropChan := make(chan *DropBlameChild, 10) wg := &sync.WaitGroup{} wg.Add(1) - // execute the deletes in a seperate single channel + // Execute the deletes in a separate goroutine go func(dC <-chan *DropBlameChild) { for elem := range dC { elem.Exec() @@ -43,18 +48,25 @@ func (p *BlameConfigProcessor) Run(ctx context.Context, e Entry, pool pool.Virtu }(dropChan) blameTask := NewBlameConfigTask(e, dropChan, p.config) - err := pool.Submit(blameTask) - if err != nil { + if err := pool.Submit(blameTask); err != nil { + // Clean up pool and channels even on early error + pool.CloseAndWait() + close(dropChan) + wg.Wait() return nil, err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() - // close the dropChan channel + + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Close the dropChan channel and wait for cleanup goroutine close(dropChan) wg.Wait() + // Return first error for fail-fast mode, or combined errors for tolerant mode + if errs := pool.Errors(); len(errs) > 0 { + return blameTask.self, errors.Join(errs...) + } return blameTask.self, pool.FirstError() } @@ -109,7 +121,7 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) child := &sdcpb.BlameTreeElement{Name: childEntry.PathName()} t.self.AddChild(child) - // create a new task for each child + // Create a new task for each child task := &BlameConfigTask{ config: t.config, parent: t.self, @@ -117,7 +129,7 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) selfEntry: childEntry, dropChan: t.dropChan, } - // submit the task + // Submit may fail if pool is closed or fail-fast error occurred if err := submit(task); err != nil { return err } diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index 45a941a5..007ce747 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -153,7 +153,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { treeRoot := tt.r(t) sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - vPool := sharedPool.NewVirtualPool(pool.VirtualFailFast, 10) + vPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) bp := NewBlameConfigProcessor(NewBlameConfigProcessorConfig(tt.includeDefaults)) got, err := bp.Run(ctx, treeRoot.sharedEntryAttributes, vPool) diff --git a/pkg/tree/processor_error_collection_test.go b/pkg/tree/processor_error_collection_test.go new file mode 100644 index 00000000..a0cb142f --- /dev/null +++ b/pkg/tree/processor_error_collection_test.go @@ -0,0 +1,107 @@ +package tree + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/sdcio/data-server/pkg/pool" +) + +// TestProcessorErrorCollection verifies that processor error collection happens +// AFTER all tasks complete, not before. This ensures CloseAndWait is called +// before checking pool.Errors(). +func TestProcessorErrorCollection(t *testing.T) { + // Create a shared task pool + ctx := context.Background() + sharedPool := pool.NewSharedTaskPool(ctx, 2) + + // Test with tolerant mode to collect multiple errors + vpool := sharedPool.NewVirtualPool(pool.VirtualTolerant) + + var errorCount atomic.Int32 + var taskCompletionCount atomic.Int32 + + // Submit tasks that will error with a delay + // This simulates tasks that take time to complete and add errors + for i := 0; i < 5; i++ { + err := vpool.Submit(pool.TaskFunc(func(ctx context.Context, submit func(pool.Task) error) error { + // Simulate some work + time.Sleep(10 * time.Millisecond) + taskCompletionCount.Add(1) + errorCount.Add(1) + return errors.New("task error") + })) + if err != nil { + t.Fatalf("Failed to submit task: %v", err) + } + } + + // Close and wait - this MUST complete before we check errors + vpool.CloseAndWait() + + // Now verify all errors are collected + errs := vpool.Errors() + + // All 5 tasks should have completed + if count := taskCompletionCount.Load(); count != 5 { + t.Errorf("Expected 5 tasks to complete, got %d", count) + } + + // All 5 errors should be collected + if len(errs) != 5 { + t.Errorf("Expected 5 errors, got %d", len(errs)) + } + + // Verify combined error works + combinedErr := errors.Join(errs...) + if combinedErr == nil { + t.Error("Expected combined error to be non-nil") + } + + sharedPool.CloseForSubmit() + sharedPool.Wait() +} + +// TestProcessorEarlyReturnCleanup verifies that if Submit fails early, +// the pool is still properly cleaned up. +func TestProcessorEarlyReturnCleanup(t *testing.T) { + ctx := context.Background() + sharedPool := pool.NewSharedTaskPool(ctx, 2) + + vpool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + + // Close the pool immediately so Submit will fail + vpool.CloseForSubmit() + + // Try to submit - this should fail + err := vpool.Submit(pool.TaskFunc(func(ctx context.Context, submit func(pool.Task) error) error { + return nil + })) + + if err == nil { + t.Fatal("Expected Submit to fail on closed pool") + } + + // Even though Submit failed, we should be able to call CloseAndWait safely + vpool.CloseAndWait() + + // Wait should not block + done := make(chan struct{}) + go func() { + vpool.Wait() + close(done) + }() + + select { + case <-done: + // Success - Wait unblocked + case <-time.After(1 * time.Second): + t.Fatal("Wait blocked after CloseAndWait on failed Submit") + } + + sharedPool.CloseForSubmit() + sharedPool.Wait() +} diff --git a/pkg/tree/processor_mark_owner_delete.go b/pkg/tree/processor_mark_owner_delete.go index 2dcf5c49..47b77f5c 100644 --- a/pkg/tree/processor_mark_owner_delete.go +++ b/pkg/tree/processor_mark_owner_delete.go @@ -2,6 +2,7 @@ package tree import ( "context" + "errors" "sync" "github.com/sdcio/data-server/pkg/pool" @@ -19,17 +20,23 @@ func NewOwnerDeleteMarker(c *OwnerDeleteMarkerTaskConfig) *MarkOwnerDeleteProces } } -func (o *MarkOwnerDeleteProcessor) Run(e Entry, pool pool.VirtualPoolI) error { - - err := pool.Submit(newOwnerDeleteMarkerTask(o.config, e, o.matches)) - if err != nil { +// Run processes the entry tree starting from e, marking leaf variant entries for deletion +// by the specified owner. The pool parameter should be VirtualFailFast to stop on first error. +// Returns the first error encountered, or nil if successful. +func (p *MarkOwnerDeleteProcessor) Run(e Entry, pool pool.VirtualPoolI) error { + if err := pool.Submit(newOwnerDeleteMarkerTask(p.config, e, p.matches)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() return err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + if errs := pool.Errors(); len(errs) > 0 { + return errors.Join(errs...) + } return pool.FirstError() } @@ -67,8 +74,12 @@ func (x ownerDeleteMarkerTask) Run(ctx context.Context, submit func(pool.Task) e if le != nil { x.matches.Append(le) } + // Process children recursively for _, c := range x.e.GetChilds(DescendMethodAll) { - submit(newOwnerDeleteMarkerTask(x.config, c, x.matches)) + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(newOwnerDeleteMarkerTask(x.config, c, x.matches)); err != nil { + return err + } } return nil } diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go index cce338a3..f1715a45 100644 --- a/pkg/tree/processor_remove_deleted.go +++ b/pkg/tree/processor_remove_deleted.go @@ -2,6 +2,7 @@ package tree import ( "context" + "errors" "sync" "sync/atomic" @@ -43,16 +44,24 @@ func (r *RemoveDeletedProcessorParameters) GetZeroLengthLeafVariantEntries() []E return r.zeroLeafEntryElements } -func (o *RemoveDeletedProcessor) Run(e Entry, pool pool.VirtualPoolI) error { - err := pool.Submit(newRemoveDeletedTask(o.config, e, false)) - if err != nil { +// Run processes the entry tree starting from e, removing leaf variant entries marked +// for deletion by the specified owner. The pool parameter should be VirtualFailFast +// to stop on first error. +// Returns the first error encountered, or nil if successful. +func (p *RemoveDeletedProcessor) Run(e Entry, pool pool.VirtualPoolI) error { + if err := pool.Submit(newRemoveDeletedTask(p.config, e, false)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() return err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + if errs := pool.Errors(); len(errs) > 0 { + return errors.Join(errs...) + } return pool.FirstError() } @@ -89,11 +98,11 @@ func (t *removeDeletedTask) Run(ctx context.Context, submit func(pool.Task) erro return nil } - // process childs + // Process children recursively for _, c := range t.e.GetChilds(DescendMethodAll) { childTask := newRemoveDeletedTask(t.config, c, t.e.GetSchema().GetContainer() == nil) - err := submit(childTask) - if err != nil { + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(childTask); err != nil { return err } } diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go index 5275199e..c12c1bd8 100644 --- a/pkg/tree/processor_reset_flags.go +++ b/pkg/tree/processor_reset_flags.go @@ -2,6 +2,7 @@ package tree import ( "context" + "errors" "fmt" "sync/atomic" @@ -38,20 +39,27 @@ func (r *ResetFlagsProcessorParameters) GetAdjustedFlagsCount() int64 { return r.adjustedFlagsCount.Load() } -func (o *ResetFlagsProcessor) Run(e Entry, pool pool.VirtualPoolI) error { +// Run processes the entry tree starting from e, resetting flags on all leaf variant entries +// according to the processor configuration. The pool parameter can be either VirtualFailFast +// (stops on first error) or VirtualTolerant (collects all errors). +// Returns the first error for fail-fast pools, or a combined error for tolerant pools. +func (p *ResetFlagsProcessor) Run(e Entry, pool pool.VirtualPoolI) error { if e == nil { return fmt.Errorf("entry cannot be nil") } - err := pool.Submit(newResetFlagsTask(o.config, e)) - if err != nil { + + // Submit root task; workers will recursively process children + if err := pool.Submit(newResetFlagsTask(p.config, e)); err != nil { + // Clean up pool even on early error + pool.CloseAndWait() return err } - // close pool for additional external submission - pool.CloseForSubmit() - // wait for the pool to run dry - pool.Wait() - return pool.FirstError() + // Close pool and wait for all tasks to complete before checking errors + pool.CloseAndWait() + + // Return first error for fail-fast mode, or combined errors for tolerant mode + return errors.Join(pool.Errors()...) } type resetFlagsTask struct { @@ -67,14 +75,14 @@ func newResetFlagsTask(config *ResetFlagsProcessorParameters, e Entry) *resetFla } func (t *resetFlagsTask) Run(ctx context.Context, submit func(pool.Task) error) error { - // reset flags as per config + // Reset flags as per config count := t.e.GetLeafVariantEntries().ResetFlags(t.config.deleteFlag, t.config.newFlag, t.config.updateFlag) t.config.adjustedFlagsCount.Add(int64(count)) - // process childs + // Process children recursively for _, c := range t.e.GetChilds(DescendMethodAll) { - err := submit(newResetFlagsTask(t.config, c)) - if err != nil { + // Submit may fail if pool is closed or fail-fast error occurred + if err := submit(newResetFlagsTask(t.config, c)); err != nil { return err } } diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go index 876e779d..929c13f2 100644 --- a/pkg/tree/processor_reset_flags_test.go +++ b/pkg/tree/processor_reset_flags_test.go @@ -101,7 +101,7 @@ func TestResetFlagsProcessorRun(t *testing.T) { // Create a virtual pool for testing taskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - virtualPool := taskPool.NewVirtualPool(pool.VirtualFailFast, 1) + virtualPool := taskPool.NewVirtualPool(pool.VirtualFailFast) root := tt.tree() diff --git a/pkg/tree/processor_validate.go b/pkg/tree/processor_validate.go index a8a018c5..4d508a47 100644 --- a/pkg/tree/processor_validate.go +++ b/pkg/tree/processor_validate.go @@ -19,10 +19,9 @@ func NewValidateProcessor(parameters *ValidateProcessorParameters) *ValidateProc } func (p *ValidateProcessor) Run(taskpoolFactory pool.VirtualPoolFactory, e Entry) { - taskpool := taskpoolFactory.NewVirtualPool(pool.VirtualTolerant, 0) + taskpool := taskpoolFactory.NewVirtualPool(pool.VirtualTolerant) taskpool.Submit(newValidateTask(e, p.parameters)) - taskpool.CloseForSubmit() - taskpool.Wait() + taskpool.CloseAndWait() } type ValidateProcessorParameters struct { diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index ce550633..ea1f8187 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -1138,7 +1138,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { // mark owner delete sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index bd73bb88..398ff87e 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -594,7 +594,7 @@ func TestToXMLTable(t *testing.T) { if tt.newConfig != nil { sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast, 1) + deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner, false)) err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) From de38550af0f669989f9224fb20a3ffeb15342098 Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 26 Jan 2026 11:12:40 +0100 Subject: [PATCH 05/17] working darft for revertives --- mocks/mockcacheclient/clientbound.go | 183 ++++++++++++++++++++++++++ mocks/mockschema/client.go | 42 +++--- mocks/mockschemaclientbound/client.go | 16 +-- mocks/mocktarget/target.go | 18 +-- mocks/mocktreeentry/entry.go | 124 ++++++++++++++--- pkg/datastore/deviations.go | 6 +- pkg/datastore/sync.go | 78 ++++++++++- pkg/datastore/sync_test.go | 98 ++++++++++---- pkg/tree/leaf_variants.go | 40 +++++- 9 files changed, 519 insertions(+), 86 deletions(-) create mode 100644 mocks/mockcacheclient/clientbound.go diff --git a/mocks/mockcacheclient/clientbound.go b/mocks/mockcacheclient/clientbound.go new file mode 100644 index 00000000..4682d5b6 --- /dev/null +++ b/mocks/mockcacheclient/clientbound.go @@ -0,0 +1,183 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/cache/cacheClientBound.go +// +// Generated by this command: +// +// mockgen -package=mockcacheclient -source=pkg/cache/cacheClientBound.go -destination=./mocks/mockcacheclient/clientbound.go +// + +// Package mockcacheclient is a generated GoMock package. +package mockcacheclient + +import ( + context "context" + reflect "reflect" + + tree_persist "github.com/sdcio/sdc-protos/tree_persist" + gomock "go.uber.org/mock/gomock" +) + +// MockCacheClientBound is a mock of CacheClientBound interface. +type MockCacheClientBound struct { + ctrl *gomock.Controller + recorder *MockCacheClientBoundMockRecorder + isgomock struct{} +} + +// MockCacheClientBoundMockRecorder is the mock recorder for MockCacheClientBound. +type MockCacheClientBoundMockRecorder struct { + mock *MockCacheClientBound +} + +// NewMockCacheClientBound creates a new mock instance. +func NewMockCacheClientBound(ctrl *gomock.Controller) *MockCacheClientBound { + mock := &MockCacheClientBound{ctrl: ctrl} + mock.recorder = &MockCacheClientBoundMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCacheClientBound) EXPECT() *MockCacheClientBoundMockRecorder { + return m.recorder +} + +// InstanceClose mocks base method. +func (m *MockCacheClientBound) InstanceClose(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceClose", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceClose indicates an expected call of InstanceClose. +func (mr *MockCacheClientBoundMockRecorder) InstanceClose(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceClose", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceClose), ctx) +} + +// InstanceCreate mocks base method. +func (m *MockCacheClientBound) InstanceCreate(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceCreate", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceCreate indicates an expected call of InstanceCreate. +func (mr *MockCacheClientBoundMockRecorder) InstanceCreate(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceCreate", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceCreate), ctx) +} + +// InstanceDelete mocks base method. +func (m *MockCacheClientBound) InstanceDelete(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceDelete", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstanceDelete indicates an expected call of InstanceDelete. +func (mr *MockCacheClientBoundMockRecorder) InstanceDelete(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceDelete", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceDelete), ctx) +} + +// InstanceExists mocks base method. +func (m *MockCacheClientBound) InstanceExists(ctx context.Context) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceExists", ctx) + ret0, _ := ret[0].(bool) + return ret0 +} + +// InstanceExists indicates an expected call of InstanceExists. +func (mr *MockCacheClientBoundMockRecorder) InstanceExists(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceExists", reflect.TypeOf((*MockCacheClientBound)(nil).InstanceExists), ctx) +} + +// IntentDelete mocks base method. +func (m *MockCacheClientBound) IntentDelete(ctx context.Context, intentName string, IgnoreNonExisting bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentDelete", ctx, intentName, IgnoreNonExisting) + ret0, _ := ret[0].(error) + return ret0 +} + +// IntentDelete indicates an expected call of IntentDelete. +func (mr *MockCacheClientBoundMockRecorder) IntentDelete(ctx, intentName, IgnoreNonExisting any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentDelete", reflect.TypeOf((*MockCacheClientBound)(nil).IntentDelete), ctx, intentName, IgnoreNonExisting) +} + +// IntentExists mocks base method. +func (m *MockCacheClientBound) IntentExists(ctx context.Context, intentName string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentExists", ctx, intentName) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentExists indicates an expected call of IntentExists. +func (mr *MockCacheClientBoundMockRecorder) IntentExists(ctx, intentName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentExists", reflect.TypeOf((*MockCacheClientBound)(nil).IntentExists), ctx, intentName) +} + +// IntentGet mocks base method. +func (m *MockCacheClientBound) IntentGet(ctx context.Context, intentName string) (*tree_persist.Intent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentGet", ctx, intentName) + ret0, _ := ret[0].(*tree_persist.Intent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentGet indicates an expected call of IntentGet. +func (mr *MockCacheClientBoundMockRecorder) IntentGet(ctx, intentName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentGet", reflect.TypeOf((*MockCacheClientBound)(nil).IntentGet), ctx, intentName) +} + +// IntentGetAll mocks base method. +func (m *MockCacheClientBound) IntentGetAll(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "IntentGetAll", ctx, excludeIntentNames, intentChan, errChan) +} + +// IntentGetAll indicates an expected call of IntentGetAll. +func (mr *MockCacheClientBoundMockRecorder) IntentGetAll(ctx, excludeIntentNames, intentChan, errChan any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentGetAll", reflect.TypeOf((*MockCacheClientBound)(nil).IntentGetAll), ctx, excludeIntentNames, intentChan, errChan) +} + +// IntentModify mocks base method. +func (m *MockCacheClientBound) IntentModify(ctx context.Context, intent *tree_persist.Intent) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentModify", ctx, intent) + ret0, _ := ret[0].(error) + return ret0 +} + +// IntentModify indicates an expected call of IntentModify. +func (mr *MockCacheClientBoundMockRecorder) IntentModify(ctx, intent any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentModify", reflect.TypeOf((*MockCacheClientBound)(nil).IntentModify), ctx, intent) +} + +// IntentsList mocks base method. +func (m *MockCacheClientBound) IntentsList(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IntentsList", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IntentsList indicates an expected call of IntentsList. +func (mr *MockCacheClientBoundMockRecorder) IntentsList(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IntentsList", reflect.TypeOf((*MockCacheClientBound)(nil).IntentsList), ctx) +} diff --git a/mocks/mockschema/client.go b/mocks/mockschema/client.go index 437e77c8..3cb356b1 100644 --- a/mocks/mockschema/client.go +++ b/mocks/mockschema/client.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" grpc "google.golang.org/grpc" ) @@ -43,14 +43,14 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // CreateSchema mocks base method. -func (m *MockClient) CreateSchema(ctx context.Context, in *schema_server.CreateSchemaRequest, opts ...grpc.CallOption) (*schema_server.CreateSchemaResponse, error) { +func (m *MockClient) CreateSchema(ctx context.Context, in *sdcpb.CreateSchemaRequest, opts ...grpc.CallOption) (*sdcpb.CreateSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CreateSchema", varargs...) - ret0, _ := ret[0].(*schema_server.CreateSchemaResponse) + ret0, _ := ret[0].(*sdcpb.CreateSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -63,14 +63,14 @@ func (mr *MockClientMockRecorder) CreateSchema(ctx, in any, opts ...any) *gomock } // DeleteSchema mocks base method. -func (m *MockClient) DeleteSchema(ctx context.Context, in *schema_server.DeleteSchemaRequest, opts ...grpc.CallOption) (*schema_server.DeleteSchemaResponse, error) { +func (m *MockClient) DeleteSchema(ctx context.Context, in *sdcpb.DeleteSchemaRequest, opts ...grpc.CallOption) (*sdcpb.DeleteSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "DeleteSchema", varargs...) - ret0, _ := ret[0].(*schema_server.DeleteSchemaResponse) + ret0, _ := ret[0].(*sdcpb.DeleteSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -83,14 +83,14 @@ func (mr *MockClientMockRecorder) DeleteSchema(ctx, in any, opts ...any) *gomock } // ExpandPath mocks base method. -func (m *MockClient) ExpandPath(ctx context.Context, in *schema_server.ExpandPathRequest, opts ...grpc.CallOption) (*schema_server.ExpandPathResponse, error) { +func (m *MockClient) ExpandPath(ctx context.Context, in *sdcpb.ExpandPathRequest, opts ...grpc.CallOption) (*sdcpb.ExpandPathResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ExpandPath", varargs...) - ret0, _ := ret[0].(*schema_server.ExpandPathResponse) + ret0, _ := ret[0].(*sdcpb.ExpandPathResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -103,14 +103,14 @@ func (mr *MockClientMockRecorder) ExpandPath(ctx, in any, opts ...any) *gomock.C } // GetSchema mocks base method. -func (m *MockClient) GetSchema(ctx context.Context, in *schema_server.GetSchemaRequest, opts ...grpc.CallOption) (*schema_server.GetSchemaResponse, error) { +func (m *MockClient) GetSchema(ctx context.Context, in *sdcpb.GetSchemaRequest, opts ...grpc.CallOption) (*sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchema", varargs...) - ret0, _ := ret[0].(*schema_server.GetSchemaResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -123,14 +123,14 @@ func (mr *MockClientMockRecorder) GetSchema(ctx, in any, opts ...any) *gomock.Ca } // GetSchemaDetails mocks base method. -func (m *MockClient) GetSchemaDetails(ctx context.Context, in *schema_server.GetSchemaDetailsRequest, opts ...grpc.CallOption) (*schema_server.GetSchemaDetailsResponse, error) { +func (m *MockClient) GetSchemaDetails(ctx context.Context, in *sdcpb.GetSchemaDetailsRequest, opts ...grpc.CallOption) (*sdcpb.GetSchemaDetailsResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchemaDetails", varargs...) - ret0, _ := ret[0].(*schema_server.GetSchemaDetailsResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaDetailsResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -143,14 +143,14 @@ func (mr *MockClientMockRecorder) GetSchemaDetails(ctx, in any, opts ...any) *go } // GetSchemaElements mocks base method. -func (m *MockClient) GetSchemaElements(ctx context.Context, req *schema_server.GetSchemaRequest, opts ...grpc.CallOption) (chan *schema_server.SchemaElem, error) { +func (m *MockClient) GetSchemaElements(ctx context.Context, req *sdcpb.GetSchemaRequest, opts ...grpc.CallOption) (chan *sdcpb.SchemaElem, error) { m.ctrl.T.Helper() varargs := []any{ctx, req} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetSchemaElements", varargs...) - ret0, _ := ret[0].(chan *schema_server.SchemaElem) + ret0, _ := ret[0].(chan *sdcpb.SchemaElem) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -163,14 +163,14 @@ func (mr *MockClientMockRecorder) GetSchemaElements(ctx, req any, opts ...any) * } // ListSchema mocks base method. -func (m *MockClient) ListSchema(ctx context.Context, in *schema_server.ListSchemaRequest, opts ...grpc.CallOption) (*schema_server.ListSchemaResponse, error) { +func (m *MockClient) ListSchema(ctx context.Context, in *sdcpb.ListSchemaRequest, opts ...grpc.CallOption) (*sdcpb.ListSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ListSchema", varargs...) - ret0, _ := ret[0].(*schema_server.ListSchemaResponse) + ret0, _ := ret[0].(*sdcpb.ListSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -183,14 +183,14 @@ func (mr *MockClientMockRecorder) ListSchema(ctx, in any, opts ...any) *gomock.C } // ReloadSchema mocks base method. -func (m *MockClient) ReloadSchema(ctx context.Context, in *schema_server.ReloadSchemaRequest, opts ...grpc.CallOption) (*schema_server.ReloadSchemaResponse, error) { +func (m *MockClient) ReloadSchema(ctx context.Context, in *sdcpb.ReloadSchemaRequest, opts ...grpc.CallOption) (*sdcpb.ReloadSchemaResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ReloadSchema", varargs...) - ret0, _ := ret[0].(*schema_server.ReloadSchemaResponse) + ret0, _ := ret[0].(*sdcpb.ReloadSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -203,14 +203,14 @@ func (mr *MockClientMockRecorder) ReloadSchema(ctx, in any, opts ...any) *gomock } // ToPath mocks base method. -func (m *MockClient) ToPath(ctx context.Context, in *schema_server.ToPathRequest, opts ...grpc.CallOption) (*schema_server.ToPathResponse, error) { +func (m *MockClient) ToPath(ctx context.Context, in *sdcpb.ToPathRequest, opts ...grpc.CallOption) (*sdcpb.ToPathResponse, error) { m.ctrl.T.Helper() varargs := []any{ctx, in} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ToPath", varargs...) - ret0, _ := ret[0].(*schema_server.ToPathResponse) + ret0, _ := ret[0].(*sdcpb.ToPathResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -223,14 +223,14 @@ func (mr *MockClientMockRecorder) ToPath(ctx, in any, opts ...any) *gomock.Call } // UploadSchema mocks base method. -func (m *MockClient) UploadSchema(ctx context.Context, opts ...grpc.CallOption) (schema_server.SchemaServer_UploadSchemaClient, error) { +func (m *MockClient) UploadSchema(ctx context.Context, opts ...grpc.CallOption) (sdcpb.SchemaServer_UploadSchemaClient, error) { m.ctrl.T.Helper() varargs := []any{ctx} for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "UploadSchema", varargs...) - ret0, _ := ret[0].(schema_server.SchemaServer_UploadSchemaClient) + ret0, _ := ret[0].(sdcpb.SchemaServer_UploadSchemaClient) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mockschemaclientbound/client.go b/mocks/mockschemaclientbound/client.go index 7a5ac461..7465f246 100644 --- a/mocks/mockschemaclientbound/client.go +++ b/mocks/mockschemaclientbound/client.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" ) @@ -42,25 +42,25 @@ func (m *MockSchemaClientBound) EXPECT() *MockSchemaClientBoundMockRecorder { } // GetSchemaElements mocks base method. -func (m *MockSchemaClientBound) GetSchemaElements(ctx context.Context, p *schema_server.Path, done chan struct{}) (chan *schema_server.GetSchemaResponse, error) { +func (m *MockSchemaClientBound) GetSchemaElements(ctx context.Context, path *sdcpb.Path, done chan struct{}) (chan *sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSchemaElements", ctx, p, done) - ret0, _ := ret[0].(chan *schema_server.GetSchemaResponse) + ret := m.ctrl.Call(m, "GetSchemaElements", ctx, path, done) + ret0, _ := ret[0].(chan *sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } // GetSchemaElements indicates an expected call of GetSchemaElements. -func (mr *MockSchemaClientBoundMockRecorder) GetSchemaElements(ctx, p, done any) *gomock.Call { +func (mr *MockSchemaClientBoundMockRecorder) GetSchemaElements(ctx, path, done any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaElements", reflect.TypeOf((*MockSchemaClientBound)(nil).GetSchemaElements), ctx, p, done) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaElements", reflect.TypeOf((*MockSchemaClientBound)(nil).GetSchemaElements), ctx, path, done) } // GetSchemaSdcpbPath mocks base method. -func (m *MockSchemaClientBound) GetSchemaSdcpbPath(ctx context.Context, path *schema_server.Path) (*schema_server.GetSchemaResponse, error) { +func (m *MockSchemaClientBound) GetSchemaSdcpbPath(ctx context.Context, path *sdcpb.Path) (*sdcpb.GetSchemaResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSchemaSdcpbPath", ctx, path) - ret0, _ := ret[0].(*schema_server.GetSchemaResponse) + ret0, _ := ret[0].(*sdcpb.GetSchemaResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mocktarget/target.go b/mocks/mocktarget/target.go index 6f7ffbeb..1fccdd01 100644 --- a/mocks/mocktarget/target.go +++ b/mocks/mocktarget/target.go @@ -15,7 +15,7 @@ import ( config "github.com/sdcio/data-server/pkg/config" types "github.com/sdcio/data-server/pkg/datastore/target/types" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" gomock "go.uber.org/mock/gomock" ) @@ -63,24 +63,24 @@ func (mr *MockTargetMockRecorder) AddSyncs(ctx any, sps ...any) *gomock.Call { } // Close mocks base method. -func (m *MockTarget) Close() error { +func (m *MockTarget) Close(ctx context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") + ret := m.ctrl.Call(m, "Close", ctx) ret0, _ := ret[0].(error) return ret0 } // Close indicates an expected call of Close. -func (mr *MockTargetMockRecorder) Close() *gomock.Call { +func (mr *MockTargetMockRecorder) Close(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTarget)(nil).Close)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTarget)(nil).Close), ctx) } // Get mocks base method. -func (m *MockTarget) Get(ctx context.Context, req *schema_server.GetDataRequest) (*schema_server.GetDataResponse, error) { +func (m *MockTarget) Get(ctx context.Context, req *sdcpb.GetDataRequest) (*sdcpb.GetDataResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", ctx, req) - ret0, _ := ret[0].(*schema_server.GetDataResponse) + ret0, _ := ret[0].(*sdcpb.GetDataResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -92,10 +92,10 @@ func (mr *MockTargetMockRecorder) Get(ctx, req any) *gomock.Call { } // Set mocks base method. -func (m *MockTarget) Set(ctx context.Context, source types.TargetSource) (*schema_server.SetDataResponse, error) { +func (m *MockTarget) Set(ctx context.Context, source types.TargetSource) (*sdcpb.SetDataResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Set", ctx, source) - ret0, _ := ret[0].(*schema_server.SetDataResponse) + ret0, _ := ret[0].(*sdcpb.SetDataResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/mocks/mocktreeentry/entry.go b/mocks/mocktreeentry/entry.go index ee8bec50..ccc82fc1 100644 --- a/mocks/mocktreeentry/entry.go +++ b/mocks/mocktreeentry/entry.go @@ -18,7 +18,7 @@ import ( tree "github.com/sdcio/data-server/pkg/tree" importer "github.com/sdcio/data-server/pkg/tree/importer" types "github.com/sdcio/data-server/pkg/tree/types" - schema_server "github.com/sdcio/sdc-protos/sdcpb" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" tree_persist "github.com/sdcio/sdc-protos/tree_persist" gomock "go.uber.org/mock/gomock" ) @@ -48,7 +48,7 @@ func (m *MockEntry) EXPECT() *MockEntryMockRecorder { } // AddUpdateRecursive mocks base method. -func (m *MockEntry) AddUpdateRecursive(ctx context.Context, relativePath *schema_server.Path, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { +func (m *MockEntry) AddUpdateRecursive(ctx context.Context, relativePath *sdcpb.Path, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddUpdateRecursive", ctx, relativePath, u, flags) ret0, _ := ret[0].(tree.Entry) @@ -63,7 +63,7 @@ func (mr *MockEntryMockRecorder) AddUpdateRecursive(ctx, relativePath, u, flags } // BreadthSearch mocks base method. -func (m *MockEntry) BreadthSearch(ctx context.Context, path *schema_server.Path) ([]tree.Entry, error) { +func (m *MockEntry) BreadthSearch(ctx context.Context, path *sdcpb.Path) ([]tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BreadthSearch", ctx, path) ret0, _ := ret[0].([]tree.Entry) @@ -93,7 +93,7 @@ func (mr *MockEntryMockRecorder) DeepCopy(tc, parent any) *gomock.Call { } // DeleteBranch mocks base method. -func (m *MockEntry) DeleteBranch(ctx context.Context, path *schema_server.Path, owner string) error { +func (m *MockEntry) DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteBranch", ctx, path, owner) ret0, _ := ret[0].(error) @@ -149,6 +149,21 @@ func (mr *MockEntryMockRecorder) GetByOwner(owner, result any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByOwner", reflect.TypeOf((*MockEntry)(nil).GetByOwner), owner, result) } +// GetChild mocks base method. +func (m *MockEntry) GetChild(name string) (tree.Entry, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChild", name) + ret0, _ := ret[0].(tree.Entry) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// GetChild indicates an expected call of GetChild. +func (mr *MockEntryMockRecorder) GetChild(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChild", reflect.TypeOf((*MockEntry)(nil).GetChild), name) +} + // GetChilds mocks base method. func (m *MockEntry) GetChilds(arg0 tree.DescendMethod) tree.EntryMap { m.ctrl.T.Helper() @@ -179,15 +194,15 @@ func (mr *MockEntryMockRecorder) GetDeletes(entries, aggregatePaths any) *gomock } // GetDeviations mocks base method. -func (m *MockEntry) GetDeviations(ch chan<- *types.DeviationEntry, activeCase bool) { +func (m *MockEntry) GetDeviations(ctx context.Context, ch chan<- *types.DeviationEntry, activeCase bool) { m.ctrl.T.Helper() - m.ctrl.Call(m, "GetDeviations", ch, activeCase) + m.ctrl.Call(m, "GetDeviations", ctx, ch, activeCase) } // GetDeviations indicates an expected call of GetDeviations. -func (mr *MockEntryMockRecorder) GetDeviations(ch, activeCase any) *gomock.Call { +func (mr *MockEntryMockRecorder) GetDeviations(ctx, ch, activeCase any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviations", reflect.TypeOf((*MockEntry)(nil).GetDeviations), ch, activeCase) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviations", reflect.TypeOf((*MockEntry)(nil).GetDeviations), ctx, ch, activeCase) } // GetFirstAncestorWithSchema mocks base method. @@ -305,10 +320,10 @@ func (mr *MockEntryMockRecorder) GetRootBasedEntryChain() *gomock.Call { } // GetSchema mocks base method. -func (m *MockEntry) GetSchema() *schema_server.SchemaElem { +func (m *MockEntry) GetSchema() *sdcpb.SchemaElem { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSchema") - ret0, _ := ret[0].(*schema_server.SchemaElem) + ret0, _ := ret[0].(*sdcpb.SchemaElem) return ret0 } @@ -390,7 +405,7 @@ func (mr *MockEntryMockRecorder) NavigateLeafRef(ctx any) *gomock.Call { } // NavigateSdcpbPath mocks base method. -func (m *MockEntry) NavigateSdcpbPath(ctx context.Context, path *schema_server.Path) (tree.Entry, error) { +func (m *MockEntry) NavigateSdcpbPath(ctx context.Context, path *sdcpb.Path) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NavigateSdcpbPath", ctx, path) ret0, _ := ret[0].(tree.Entry) @@ -419,10 +434,10 @@ func (mr *MockEntryMockRecorder) PathName() *gomock.Call { } // SdcpbPath mocks base method. -func (m *MockEntry) SdcpbPath() *schema_server.Path { +func (m *MockEntry) SdcpbPath() *sdcpb.Path { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SdcpbPath") - ret0, _ := ret[0].(*schema_server.Path) + ret0, _ := ret[0].(*sdcpb.Path) return ret0 } @@ -506,16 +521,16 @@ func (mr *MockEntryMockRecorder) TreeExport(owner any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TreeExport", reflect.TypeOf((*MockEntry)(nil).TreeExport), owner) } -// Validate mocks base method. -func (m *MockEntry) Validate(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { +// ValidateLevel mocks base method. +func (m *MockEntry) ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Validate", ctx, resultChan, stats, vCfg) + m.ctrl.Call(m, "ValidateLevel", ctx, resultChan, stats, vCfg) } -// Validate indicates an expected call of Validate. -func (mr *MockEntryMockRecorder) Validate(ctx, resultChan, stats, vCfg any) *gomock.Call { +// ValidateLevel indicates an expected call of ValidateLevel. +func (mr *MockEntryMockRecorder) ValidateLevel(ctx, resultChan, stats, vCfg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Validate", reflect.TypeOf((*MockEntry)(nil).Validate), ctx, resultChan, stats, vCfg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateLevel", reflect.TypeOf((*MockEntry)(nil).ValidateLevel), ctx, resultChan, stats, vCfg) } // Walk mocks base method. @@ -546,6 +561,21 @@ func (mr *MockEntryMockRecorder) addChild(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addChild", reflect.TypeOf((*MockEntry)(nil).addChild), arg0, arg1) } +// addUpdateRecursiveInternal mocks base method. +func (m *MockEntry) addUpdateRecursiveInternal(ctx context.Context, path *sdcpb.Path, idx int, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "addUpdateRecursiveInternal", ctx, path, idx, u, flags) + ret0, _ := ret[0].(tree.Entry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// addUpdateRecursiveInternal indicates an expected call of addUpdateRecursiveInternal. +func (mr *MockEntryMockRecorder) addUpdateRecursiveInternal(ctx, path, idx, u, flags any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addUpdateRecursiveInternal", reflect.TypeOf((*MockEntry)(nil).addUpdateRecursiveInternal), ctx, path, idx, u, flags) +} + // canDelete mocks base method. func (m *MockEntry) canDelete() bool { m.ctrl.T.Helper() @@ -616,7 +646,7 @@ func (mr *MockEntryMockRecorder) getHighestPrecedenceValueOfBranch(filter any) * } // getOrCreateChilds mocks base method. -func (m *MockEntry) getOrCreateChilds(ctx context.Context, path *schema_server.Path) (tree.Entry, error) { +func (m *MockEntry) getOrCreateChilds(ctx context.Context, path *sdcpb.Path) (tree.Entry, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "getOrCreateChilds", ctx, path) ret0, _ := ret[0].(tree.Entry) @@ -866,6 +896,18 @@ func (m *MockLeafVariantEntries) EXPECT() *MockLeafVariantEntriesMockRecorder { return m.recorder } +// Add mocks base method. +func (m *MockLeafVariantEntries) Add(l *tree.LeafEntry) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Add", l) +} + +// Add indicates an expected call of Add. +func (mr *MockLeafVariantEntriesMockRecorder) Add(l any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockLeafVariantEntries)(nil).Add), l) +} + // AddExplicitDeleteEntry mocks base method. func (m *MockLeafVariantEntries) AddExplicitDeleteEntry(owner string, priority int32) *tree.LeafEntry { m.ctrl.T.Helper() @@ -936,6 +978,20 @@ func (mr *MockLeafVariantEntriesMockRecorder) GetRunning() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunning", reflect.TypeOf((*MockLeafVariantEntries)(nil).GetRunning)) } +// Length mocks base method. +func (m *MockLeafVariantEntries) Length() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Length") + ret0, _ := ret[0].(int) + return ret0 +} + +// Length indicates an expected call of Length. +func (mr *MockLeafVariantEntriesMockRecorder) Length() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Length", reflect.TypeOf((*MockLeafVariantEntries)(nil).Length)) +} + // MarkOwnerForDeletion mocks base method. func (m *MockLeafVariantEntries) MarkOwnerForDeletion(owner string, onlyIntended bool) *tree.LeafEntry { m.ctrl.T.Helper() @@ -949,3 +1005,31 @@ func (mr *MockLeafVariantEntriesMockRecorder) MarkOwnerForDeletion(owner, onlyIn mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkOwnerForDeletion", reflect.TypeOf((*MockLeafVariantEntries)(nil).MarkOwnerForDeletion), owner, onlyIntended) } + +// RemoveDeletedByOwner mocks base method. +func (m *MockLeafVariantEntries) RemoveDeletedByOwner(owner string) *tree.LeafEntry { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveDeletedByOwner", owner) + ret0, _ := ret[0].(*tree.LeafEntry) + return ret0 +} + +// RemoveDeletedByOwner indicates an expected call of RemoveDeletedByOwner. +func (mr *MockLeafVariantEntriesMockRecorder) RemoveDeletedByOwner(owner any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDeletedByOwner", reflect.TypeOf((*MockLeafVariantEntries)(nil).RemoveDeletedByOwner), owner) +} + +// ResetFlags mocks base method. +func (m *MockLeafVariantEntries) ResetFlags(deleteFlag, newFlag, updatedFlag bool) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetFlags", deleteFlag, newFlag, updatedFlag) + ret0, _ := ret[0].(int) + return ret0 +} + +// ResetFlags indicates an expected call of ResetFlags. +func (mr *MockLeafVariantEntriesMockRecorder) ResetFlags(deleteFlag, newFlag, updatedFlag any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetFlags", reflect.TypeOf((*MockLeafVariantEntries)(nil).ResetFlags), deleteFlag, newFlag, updatedFlag) +} diff --git a/pkg/datastore/deviations.go b/pkg/datastore/deviations.go index 7681e3c9..b3bbb1ee 100644 --- a/pkg/datastore/deviations.go +++ b/pkg/datastore/deviations.go @@ -125,7 +125,11 @@ func (d *Datastore) SendDeviations(ctx context.Context, ch <-chan *treetypes.Dev if log := log.V(logger.VTrace); log.Enabled() { if deviation.Reason() == treetypes.DeviationReasonNotApplied { // TODO add check for trace level Trace - log.Info("NOT APPLIED", "path", deviation.Path().ToXPath(false), "actual value", deviation.CurrentValue().ToString(), "expected value", deviation.ExpectedValue().ToString(), "intent", deviation.IntentName()) + current := "nil" + if deviation.CurrentValue() != nil { + current = deviation.CurrentValue().ToString() + } + log.Info("NOT APPLIED", "path", deviation.Path().ToXPath(false), "actual value", current, "expected value", deviation.ExpectedValue().ToString(), "intent", deviation.IntentName()) } } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 93760ac8..cb87a8a0 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -3,6 +3,7 @@ package datastore import ( "context" "errors" + "sync" "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" @@ -18,7 +19,9 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i log := logger.FromContext(ctx) d.syncTreeMutex.Lock() - defer d.syncTreeMutex.Unlock() + syncTreeUnlock := sync.OnceFunc(d.syncTreeMutex.Unlock) + + defer syncTreeUnlock() // create a virtual task pool for delete operations deleteMarkerPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) @@ -88,7 +91,37 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } } + // run reset flags processor to reset flags + resetFlagsPool := d.taskPool.NewVirtualPool(pool.VirtualTolerant) + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(true, true, true) + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(d.syncTree.GetRoot(), resetFlagsPool) + if err != nil { + return err + } + + // close the resetFlags pool for submission and wait + resetFlagsPool.CloseAndWait() + if errors.Join(resetFlagsPool.Errors()...) != nil { + return err + } + + syncTreeCopy, err := d.syncTree.DeepCopy(ctx) + if err != nil { + return err + } + + // release the sync tree lock early, it is no longer needed + syncTreeUnlock() + + // perform the revert operation to apply changes to the device + // TODO: this should probably be executed in a separate goroutine + err = d.performRevert(ctx, syncTreeCopy) + if err != nil { + return err + } + return nil + } func (d *Datastore) NewEmptyTree(ctx context.Context) (*tree.RootEntry, error) { @@ -99,3 +132,46 @@ func (d *Datastore) NewEmptyTree(ctx context.Context) (*tree.RootEntry, error) { } return newTree, nil } + +func (d *Datastore) performRevert(ctx context.Context, t *tree.RootEntry) error { + log := logger.FromContext(ctx) + _, err := d.LoadAllButRunningIntents(ctx, t) + if err != nil { + return err + } + + err = t.FinishInsertionPhase(ctx) + if err != nil { + return err + } + + // TODO: optimize by checking only paths that where covered by the syncconfig + del, err := t.GetDeletes(true) + if err != nil { + return err + } + + // if we have deletes, we need to perform an apply + performApply := len(del) > 0 + + // if no deletes, check if we have updates + if !performApply { + updList, err := t.ToProtoUpdates(ctx, true) + if err != nil { + return err + } + // if the update list is non-empty, we need to perform an apply + performApply = len(updList) > 0 + } + + if performApply { + resp, err := d.applyIntent(ctx, t) + if err != nil { + respJ := protojson.MarshalOptions{Multiline: false} + respStr, _ := respJ.Marshal(resp) + log.Error(err, "failed applying deviations to running", "response", string(respStr)) + } + } + + return nil +} diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index fe9701b2..5ef61d4b 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -10,7 +10,10 @@ import ( "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/mocks/mockcacheclient" + "github.com/sdcio/data-server/mocks/mocktarget" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/datastore/target" "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" @@ -19,6 +22,8 @@ import ( "github.com/sdcio/data-server/pkg/utils/testhelper" sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" + "go.uber.org/mock/gomock" ) // TestApplyToRunning tests the ApplyToRunning method of the Datastore struct. @@ -28,17 +33,19 @@ func TestApplyToRunning(t *testing.T) { // Define test cases tests := []struct { - name string - deletes []*sdcpb.Path - importer func() importer.ImportConfigAdapter - syncTree func() *tree.RootEntry - result func() any - wantErr bool + name string + deletes []*sdcpb.Path + importerFunc func() importer.ImportConfigAdapter + syncTreeFunc func() *tree.RootEntry + resultFunc func() any + cacheClientFunc func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound + sbiFunc func(ctrl *gomock.Controller) target.Target + wantErr bool }{ { name: "delete entire interface (e1-1 existed now e1-2 added)", deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, - syncTree: func() *tree.RootEntry { + syncTreeFunc: func() *tree.RootEntry { ctx := context.Background() @@ -73,7 +80,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -81,7 +88,7 @@ func TestApplyToRunning(t *testing.T) { return root }, - importer: func() importer.ImportConfigAdapter { + importerFunc: func() importer.ImportConfigAdapter { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/2": { @@ -103,7 +110,7 @@ func TestApplyToRunning(t *testing.T) { return jsonImporter.NewJsonTreeImporter(v) }, - result: func() any { + resultFunc: func() any { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/2": { @@ -125,11 +132,25 @@ func TestApplyToRunning(t *testing.T) { return v }, wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, }, { name: "delete description of existing interface", deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, - syncTree: func() *tree.RootEntry { + syncTreeFunc: func() *tree.RootEntry { ctx := context.Background() @@ -164,7 +185,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -172,7 +193,7 @@ func TestApplyToRunning(t *testing.T) { return root }, - importer: func() importer.ImportConfigAdapter { + importerFunc: func() importer.ImportConfigAdapter { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { @@ -193,7 +214,7 @@ func TestApplyToRunning(t *testing.T) { return jsonImporter.NewJsonTreeImporter(v) }, - result: func() any { + resultFunc: func() any { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { @@ -214,11 +235,25 @@ func TestApplyToRunning(t *testing.T) { return v }, wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, }, { name: "change description of existing interface", deletes: []*sdcpb.Path{{Elem: []*sdcpb.PathElem{}}}, - syncTree: func() *tree.RootEntry { + syncTreeFunc: func() *tree.RootEntry { ctx := context.Background() @@ -253,7 +288,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -261,7 +296,7 @@ func TestApplyToRunning(t *testing.T) { return root }, - importer: func() importer.ImportConfigAdapter { + importerFunc: func() importer.ImportConfigAdapter { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { @@ -283,7 +318,7 @@ func TestApplyToRunning(t *testing.T) { return jsonImporter.NewJsonTreeImporter(v) }, - result: func() any { + resultFunc: func() any { d := &sdcio_schema.Device{ Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { @@ -305,6 +340,20 @@ func TestApplyToRunning(t *testing.T) { return v }, wantErr: false, + cacheClientFunc: func(ctrl *gomock.Controller) *mockcacheclient.MockCacheClientBound { + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + return ccb + }, + sbiFunc: func(ctrl *gomock.Controller) target.Target { + sbi := mocktarget.NewMockTarget(ctrl) + return sbi + }, }, } @@ -314,14 +363,19 @@ func TestApplyToRunning(t *testing.T) { fmt.Println("----" + tt.name) t.Run(tt.name, func(t *testing.T) { - syncTree := tt.syncTree() + syncTree := tt.syncTreeFunc() + + ctrl := gomock.NewController(t) datastore := &Datastore{ syncTreeMutex: &sync.RWMutex{}, syncTree: syncTree, taskPool: pool.NewSharedTaskPool(ctx, runtime.NumCPU()), + cacheClient: tt.cacheClientFunc(ctrl), + sbi: tt.sbiFunc(ctrl), } - err := datastore.ApplyToRunning(ctx, tt.deletes, tt.importer()) + + err := datastore.ApplyToRunning(ctx, tt.deletes, tt.importerFunc()) if (err != nil) != tt.wantErr { t.Errorf("ApplyToRunning() error = %v, wantErr %v", err, tt.wantErr) } @@ -340,9 +394,9 @@ func TestApplyToRunning(t *testing.T) { t.Fatalf("failed to create new tree root: %v", err) } - d := tt.result() + d := tt.resultFunc() - err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d), tree.RunningIntentName, tree.RunningValuesPrio, types.NewUpdateInsertFlags()) + err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index 9bf01699..db9806a3 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -103,6 +103,32 @@ func (lv *LeafVariants) RemoveDeletedByOwner(owner string) *LeafEntry { return nil } +// checkOnlyRunningAndMaybeDefault checks if only running and maybe default LeafVariants exist +func (lv *LeafVariants) checkOnlyRunningAndMaybeDefault() bool { + lv.lesMutex.RLock() + defer lv.lesMutex.RUnlock() + + // if we have runnig and only running we should not delete + if len(lv.les) == 1 && lv.les[0].Owner() == RunningIntentName { + return false + } + + // check if only running and default exist + hasRunning := false + hasDefault := false + if len(lv.les) == 2 { + for _, l := range lv.les { + if l.Owner() == RunningIntentName { + hasRunning = true + } + if l.Owner() == DefaultsIntentName { + hasDefault = true + } + } + } + return hasRunning && hasDefault +} + // canDelete returns true if leafValues exist that are not owned by default or running that do not have the DeleteFlag set [or if delete is set, also the DeleteOnlyIntendedFlag set] func (lv *LeafVariants) canDelete() bool { lv.lesMutex.RLock() @@ -112,8 +138,8 @@ func (lv *LeafVariants) canDelete() bool { return true } - // if we have runnig and only running we should not delete - if len(lv.les) == 1 && lv.les[0].Owner() == RunningIntentName { + // if we have runnig and only running (or default in addition) we should not delete + if lv.checkOnlyRunningAndMaybeDefault() { return false } @@ -283,6 +309,7 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau return nil } + // figure out the highest precedence LeafEntry var highest *LeafEntry var secondHighest *LeafEntry for _, e := range lv.les { @@ -337,14 +364,19 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau return nil } +// highestIsUnequalRunning checks if the highest precedence LeafEntry is unequal to the running LeafEntry +// Expects the caller to hold the read lock on lesMutex. func (lv *LeafVariants) highestIsUnequalRunning(highest *LeafEntry) bool { - lv.lesMutex.RLock() - defer lv.lesMutex.RUnlock() // if highes is already running or even default, return false if highest.Update.Owner() == RunningIntentName { return false } + // if highest is not new or updated and highest is non-revertive + if !highest.IsNew && !highest.IsUpdated && lv.tc.nonRevertiveInfo[highest.Update.Owner()] { + return false + } + runVal := lv.GetByOwner(RunningIntentName) if runVal == nil { return true From 824979877f1effa35ef192434d12da72cbc4b6be Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 26 Jan 2026 13:58:08 +0100 Subject: [PATCH 06/17] update mod --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 597b9c22..4013ff08 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/sdcio/yang-parser v0.0.12 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 @@ -79,6 +80,7 @@ require ( github.com/olekukonko/ll v0.0.9 // indirect github.com/openconfig/grpctunnel v0.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect From 053af526efe263276ae719a996d1a3aafa419e8b Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 27 Jan 2026 12:02:13 +0100 Subject: [PATCH 07/17] non-revertive now working --- pkg/datastore/intent_rpc.go | 2 +- pkg/datastore/sync.go | 4 +- pkg/datastore/sync_test.go | 14 ++--- pkg/datastore/target/gnmi/get.go | 2 +- pkg/datastore/target/gnmi/stream.go | 2 +- pkg/datastore/target/netconf/nc.go | 3 +- pkg/datastore/transaction_rpc.go | 10 ++-- pkg/datastore/tree_operation_test.go | 2 +- .../tree_operation_validation_test.go | 4 +- pkg/tree/entry.go | 3 +- pkg/tree/importer/import_config_adapter.go | 3 + pkg/tree/importer/json/json_tree_importer.go | 59 +++++++++++++------ .../importer/json/json_tree_importer_test.go | 12 ++-- .../importer/proto/proto_tree_importer.go | 31 ++++++++-- .../proto/proto_tree_importer_test.go | 8 +-- pkg/tree/importer/xml/xml_tree_importer.go | 51 ++++++++++++---- .../importer/xml/xml_tree_importer_test.go | 2 +- pkg/tree/leaf_entry.go | 10 +++- pkg/tree/leaf_variants.go | 8 ++- pkg/tree/parallelImporter.go | 6 +- pkg/tree/root_entry.go | 19 ++++-- pkg/tree/root_entry_test.go | 4 +- pkg/tree/sharedEntryAttributes.go | 4 ++ pkg/tree/sharedEntryAttributes_test.go | 14 ++--- pkg/tree/validation_entry_leafref_test.go | 2 +- pkg/tree/validation_range_test.go | 12 ++-- pkg/utils/testhelper/utils.go | 4 +- 27 files changed, 198 insertions(+), 97 deletions(-) diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 80df1ec2..e89bfea2 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -77,7 +77,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, tp.GetIntentName(), tp.GetPriority(), tp.GetNonRevertive(), types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags()) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index cb87a8a0..c84f5f7e 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -49,7 +49,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i // import new config if provided if importer != nil { - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, tree.RunningIntentName, tree.RunningValuesPrio, false, treetypes.NewUpdateInsertFlags()) + err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags()) if err != nil { return err } @@ -82,7 +82,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i // conditional trace logging if log := log.V(logger.VTrace); log.Enabled() { - treeExport, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + treeExport, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err == nil { json, err := protojson.MarshalOptions{Multiline: false}.Marshal(treeExport) if err == nil { diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index 5ef61d4b..e45e41b5 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -80,7 +80,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -108,7 +108,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - return jsonImporter.NewJsonTreeImporter(v) + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) }, resultFunc: func() any { d := &sdcio_schema.Device{ @@ -185,7 +185,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -212,7 +212,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - return jsonImporter.NewJsonTreeImporter(v) + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) }, resultFunc: func() any { d := &sdcio_schema.Device{ @@ -288,7 +288,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -316,7 +316,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - return jsonImporter.NewJsonTreeImporter(v) + return jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false) }, resultFunc: func() any { d := &sdcio_schema.Device{ @@ -396,7 +396,7 @@ func TestApplyToRunning(t *testing.T) { d := tt.resultFunc() - err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d), tree.RunningIntentName, tree.RunningValuesPrio, false, types.NewUpdateInsertFlags()) + err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatalf("failed to import test config: %v", err) } diff --git a/pkg/datastore/target/gnmi/get.go b/pkg/datastore/target/gnmi/get.go index d76bc0ad..ab5900e1 100644 --- a/pkg/datastore/target/gnmi/get.go +++ b/pkg/datastore/target/gnmi/get.go @@ -142,7 +142,7 @@ func (s *GetSync) internalGetSync(req *sdcpb.GetDataRequest) { return } - result, err := s.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + result, err := s.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err != nil { log.Error(err, "failure exporting synctree") return diff --git a/pkg/datastore/target/gnmi/stream.go b/pkg/datastore/target/gnmi/stream.go index fd4f12af..96d42067 100644 --- a/pkg/datastore/target/gnmi/stream.go +++ b/pkg/datastore/target/gnmi/stream.go @@ -221,7 +221,7 @@ func (s *StreamSync) syncToRunning(syncTree *tree.RootEntry, m *sync.Mutex, logC defer m.Unlock() startTime := time.Now() - result, err := syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + result, err := syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) log.V(logger.VTrace).Info("exported tree", "tree", result.String()) if err != nil { diff --git a/pkg/datastore/target/netconf/nc.go b/pkg/datastore/target/netconf/nc.go index 1f77aa0f..84a62439 100644 --- a/pkg/datastore/target/netconf/nc.go +++ b/pkg/datastore/target/netconf/nc.go @@ -30,6 +30,7 @@ import ( "github.com/sdcio/data-server/pkg/datastore/target/netconf/driver/scrapligo" nctypes "github.com/sdcio/data-server/pkg/datastore/target/netconf/types" "github.com/sdcio/data-server/pkg/datastore/target/types" + "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/importer/xml" ) @@ -88,7 +89,7 @@ func (t *ncTarget) GetImportAdapter(ctx context.Context, req *sdcpb.GetDataReque return nil, err } - cmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root()) + cmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root(), tree.RunningIntentName, tree.RunningValuesPrio, false) return cmlImport, nil } diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index b6cc10d5..2f6ccef6 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -86,7 +86,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), tree.RunningIntentName, tree.RunningValuesPrio, false, treetypes.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags()) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, intent.GetIntentName(), intent.GetPriority(), intent.GetNonRevertive(), treetypes.NewUpdateInsertFlags()) + err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags()) if err != nil { return nil, err } @@ -251,6 +251,8 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ // add the explicit delete entries root.AddExplicitDeletes(intent.GetName(), intent.GetPriority(), intent.GetDeletes()) } + + root.SetNonRevertiveIntent(intent.GetName(), intent.NonRevertive()) } les := tree.LeafVariantSlice{} @@ -351,7 +353,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ delSl := deletesOwner.ToXPathSlice() log.V(logf.VTrace).Info("deletes owner", "deletes-owner", delSl) - protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority(), intent.NonRevertive()) + protoIntent, err := root.TreeExport(intent.GetName(), intent.GetPriority()) switch { case errors.Is(err, tree.ErrorIntentNotPresent): err = d.cacheClient.IntentDelete(ctx, intent.GetName(), intent.GetDeleteIgnoreNonExisting()) @@ -417,7 +419,7 @@ func (d *Datastore) writeBackSyncTree(ctx context.Context, updates tree.LeafVari } // export the synctree - newRunningIntent, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio, false) + newRunningIntent, err := d.syncTree.TreeExport(tree.RunningIntentName, tree.RunningValuesPrio) if err != nil && err != tree.ErrorIntentNotPresent { return err } diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 2ed45c7f..d31915b8 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1519,7 +1519,7 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny), tt.intentName, tt.intentPrio, tt.nonRevertive, newFlag) + err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag) if err != nil { t.Error(err) } diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index fc020302..bda39bb8 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -196,9 +196,9 @@ func TestDatastore_validateTree(t *testing.T) { flagsNew := types.NewUpdateInsertFlags() flagsNew.SetNewFlag() - importer := json_importer.NewJsonTreeImporter(jsonConf) + importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) - err = root.ImportConfig(ctx, path, importer, tt.intentName, tt.intentPrio, tt.nonRevertive, flagsNew) + err = root.ImportConfig(ctx, path, importer, flagsNew) if err != nil { t.Error(err) } diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 0238382f..94725b7e 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -131,7 +131,7 @@ type Entry interface { ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - ImportConfig(ctx context.Context, importer importer.ImportConfigAdapterElement, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error + ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) @@ -147,6 +147,7 @@ type Entry interface { HoldsLeafvariants() bool canDeleteBranch(keepDefault bool) bool deleteCanDeleteChilds(keepDefault bool) + getTreeContext() *TreeContext } type EntryVisitor interface { diff --git a/pkg/tree/importer/import_config_adapter.go b/pkg/tree/importer/import_config_adapter.go index a98f4abb..97418345 100644 --- a/pkg/tree/importer/import_config_adapter.go +++ b/pkg/tree/importer/import_config_adapter.go @@ -11,6 +11,9 @@ import ( type ImportConfigAdapter interface { ImportConfigAdapterElement GetDeletes() *sdcpb.PathSet + GetName() string + GetPriority() int32 + GetNonRevertive() bool } type ImportConfigAdapterElement interface { diff --git a/pkg/tree/importer/json/json_tree_importer.go b/pkg/tree/importer/json/json_tree_importer.go index 37e9073b..212c2416 100644 --- a/pkg/tree/importer/json/json_tree_importer.go +++ b/pkg/tree/importer/json/json_tree_importer.go @@ -12,29 +12,53 @@ import ( ) type JsonTreeImporter struct { - data any - name string + JsonTreeImporterElement + intentName string + priority int32 + nonRevertive bool +} + +func (j *JsonTreeImporter) GetPriority() int32 { + return j.priority +} + +func (j *JsonTreeImporter) GetNonRevertive() bool { + return j.nonRevertive +} + +func (j *JsonTreeImporter) GetName() string { + return j.intentName } -func newJsonTreeImporterInternal(name string, d any) *JsonTreeImporter { +func NewJsonTreeImporter(d any, intentName string, priority int32, nonRevertive bool) *JsonTreeImporter { return &JsonTreeImporter{ - data: d, - name: name, + JsonTreeImporterElement: JsonTreeImporterElement{ + data: d, + name: "root", + }, + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, } } -func NewJsonTreeImporter(d any) *JsonTreeImporter { - return &JsonTreeImporter{ +type JsonTreeImporterElement struct { + data any + name string +} + +func newJsonTreeImporterElement(name string, d any) *JsonTreeImporterElement { + return &JsonTreeImporterElement{ data: d, - name: "root", + name: name, } } -func (j *JsonTreeImporter) GetDeletes() *sdcpb.PathSet { +func (j *JsonTreeImporterElement) GetDeletes() *sdcpb.PathSet { return sdcpb.NewPathSet() } -func (j *JsonTreeImporter) GetElement(key string) importer.ImportConfigAdapterElement { +func (j *JsonTreeImporterElement) GetElement(key string) importer.ImportConfigAdapterElement { switch d := j.data.(type) { case map[string]any: @@ -44,14 +68,14 @@ func (j *JsonTreeImporter) GetElement(key string) importer.ImportConfigAdapterEl elemName = beforeColon } if key == elemName { - return newJsonTreeImporterInternal(key, v) + return newJsonTreeImporterElement(key, v) } } } return nil } -func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { +func (j *JsonTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { var result []importer.ImportConfigAdapterElement switch d := j.data.(type) { case map[string]any: @@ -64,10 +88,10 @@ func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { switch subElem := v.(type) { case []any: for _, listElem := range subElem { - result = append(result, newJsonTreeImporterInternal(key, listElem)) + result = append(result, newJsonTreeImporterElement(key, listElem)) } default: - result = append(result, newJsonTreeImporterInternal(key, v)) + result = append(result, newJsonTreeImporterElement(key, v)) } } default: @@ -76,17 +100,18 @@ func (j *JsonTreeImporter) GetElements() []importer.ImportConfigAdapterElement { return result } -func (j *JsonTreeImporter) GetKeyValue() (string, error) { +func (j *JsonTreeImporterElement) GetKeyValue() (string, error) { return fmt.Sprintf("%v", j.data), nil } -func (j *JsonTreeImporter) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { +func (j *JsonTreeImporterElement) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { return utils.ConvertJsonValueToTv(j.data, slt) } -func (j *JsonTreeImporter) GetName() string { +func (j *JsonTreeImporterElement) GetName() string { return j.name } // Function to ensure JsonTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*JsonTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*JsonTreeImporterElement)(nil) diff --git a/pkg/tree/importer/json/json_tree_importer_test.go b/pkg/tree/importer/json/json_tree_importer_test.go index 527eceae..34d0778e 100644 --- a/pkg/tree/importer/json/json_tree_importer_test.go +++ b/pkg/tree/importer/json/json_tree_importer_test.go @@ -20,7 +20,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { name string fields fields args args - want importer.ImportConfigAdapter + want importer.ImportConfigAdapterElement }{ { name: "one", @@ -32,7 +32,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { "foo": "bar", }, }, - want: &JsonTreeImporter{ + want: &JsonTreeImporterElement{ data: "bar", name: "foo", }, @@ -40,7 +40,7 @@ func TestJsonTreeImporter_GetElement(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := NewJsonTreeImporter(tt.fields.data) + j := NewJsonTreeImporter(tt.fields.data, "owner1", 5, false) if got := j.GetElement(tt.args.key); !reflect.DeepEqual(got, tt.want) { t.Errorf("JsonTreeImporter.GetElement() = %v, want %v", got, tt.want) } @@ -77,7 +77,7 @@ func TestJsonTreeImporter_GetKeyValue(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := newJsonTreeImporterInternal(tt.fields.name, tt.fields.data) + j := newJsonTreeImporterElement(tt.fields.name, tt.fields.data) if got, _ := j.GetKeyValue(); got != tt.want { t.Errorf("JsonTreeImporter.GetKeyValue() = %v, want %v", got, tt.want) @@ -106,7 +106,7 @@ func TestJsonTreeImporter_GetName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := &JsonTreeImporter{ + j := &JsonTreeImporterElement{ data: tt.fields.data, name: tt.fields.name, } @@ -164,7 +164,7 @@ func TestJsonTreeImporter_GetTVValue(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - j := &JsonTreeImporter{ + j := &JsonTreeImporterElement{ data: tt.fields.data, name: tt.fields.name, } diff --git a/pkg/tree/importer/proto/proto_tree_importer.go b/pkg/tree/importer/proto/proto_tree_importer.go index 544cd3c3..7ad01a19 100644 --- a/pkg/tree/importer/proto/proto_tree_importer.go +++ b/pkg/tree/importer/proto/proto_tree_importer.go @@ -12,7 +12,10 @@ import ( type ProtoTreeImporter struct { ProtoTreeImporterElement - deletes *sdcpb.PathSet + deletes *sdcpb.PathSet + intentName string + priority int32 + nonRevertive bool } func NewProtoTreeImporter(data *tree_persist.Intent) *ProtoTreeImporter { @@ -23,10 +26,29 @@ func NewProtoTreeImporter(data *tree_persist.Intent) *ProtoTreeImporter { ProtoTreeImporterElement: ProtoTreeImporterElement{ data: data.GetRoot(), }, - deletes: pathSet, + deletes: pathSet, + intentName: data.GetIntentName(), + priority: data.GetPriority(), + nonRevertive: data.GetNonRevertive(), } } +func (p *ProtoTreeImporter) GetPriority() int32 { + return p.priority +} + +func (p *ProtoTreeImporter) GetNonRevertive() bool { + return p.nonRevertive +} + +func (p *ProtoTreeImporter) GetName() string { + return p.intentName +} + +func (p *ProtoTreeImporter) GetDeletes() *sdcpb.PathSet { + return p.deletes +} + type ProtoTreeImporterElement struct { data *tree_persist.TreeElement } @@ -37,10 +59,6 @@ func NewProtoTreeImporterElement(data *tree_persist.TreeElement) *ProtoTreeImpor } } -func (p *ProtoTreeImporter) GetDeletes() *sdcpb.PathSet { - return p.deletes -} - func (p *ProtoTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { if p.data == nil || len(p.data.Childs) == 0 { return nil @@ -83,3 +101,4 @@ func (p *ProtoTreeImporterElement) GetName() string { // Function to ensure ProtoTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*ProtoTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*ProtoTreeImporterElement)(nil) diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 8e9a2e9a..172aed09 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -126,8 +126,8 @@ func TestProtoTreeImporter(t *testing.T) { t.Fatalf("error parsing json document: %v", err) } - jti := jimport.NewJsonTreeImporter(j) - err = root.ImportConfig(ctx, nil, jti, "owner1", 5, false, types.NewUpdateInsertFlags()) + jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) + err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags()) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func TestProtoTreeImporter(t *testing.T) { } t.Log(root.String()) - protoIntent, err := root.TreeExport("owner1", 5, false) + protoIntent, err := root.TreeExport("owner1", 5) if err != nil { t.Error(err) } @@ -153,7 +153,7 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, protoIntent.GetIntentName(), protoIntent.GetPriority(), false, types.NewUpdateInsertFlags()) + err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer.go b/pkg/tree/importer/xml/xml_tree_importer.go index 1e78b955..33e6b563 100644 --- a/pkg/tree/importer/xml/xml_tree_importer.go +++ b/pkg/tree/importer/xml/xml_tree_importer.go @@ -10,28 +10,58 @@ import ( ) type XmlTreeImporter struct { - elem *etree.Element + XmlTreeImporterElement + intentName string + priority int32 + nonRevertive bool } -func NewXmlTreeImporter(d *etree.Element) *XmlTreeImporter { +func NewXmlTreeImporter(d *etree.Element, intentName string, priority int32, nonRevertive bool) *XmlTreeImporter { return &XmlTreeImporter{ + XmlTreeImporterElement: XmlTreeImporterElement{ + elem: d, + }, + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, + } +} + +func (x *XmlTreeImporter) GetName() string { + return x.intentName +} + +func (x *XmlTreeImporter) GetPriority() int32 { + return x.priority +} + +func (x *XmlTreeImporter) GetNonRevertive() bool { + return x.nonRevertive +} + +type XmlTreeImporterElement struct { + elem *etree.Element +} + +func NewXmlTreeImporterElement(d *etree.Element) *XmlTreeImporterElement { + return &XmlTreeImporterElement{ elem: d, } } -func (x *XmlTreeImporter) GetDeletes() *sdcpb.PathSet { +func (x *XmlTreeImporterElement) GetDeletes() *sdcpb.PathSet { return sdcpb.NewPathSet() } -func (x *XmlTreeImporter) GetElement(key string) importer.ImportConfigAdapterElement { +func (x *XmlTreeImporterElement) GetElement(key string) importer.ImportConfigAdapterElement { e := x.elem.FindElement(key) if e == nil { return nil } - return NewXmlTreeImporter(e) + return NewXmlTreeImporterElement(e) } -func (x *XmlTreeImporter) GetElements() []importer.ImportConfigAdapterElement { +func (x *XmlTreeImporterElement) GetElements() []importer.ImportConfigAdapterElement { childs := x.elem.ChildElements() if len(childs) == 0 { return nil @@ -40,23 +70,24 @@ func (x *XmlTreeImporter) GetElements() []importer.ImportConfigAdapterElement { result := make([]importer.ImportConfigAdapterElement, 0, len(childs)) for _, c := range childs { - result = append(result, NewXmlTreeImporter(c)) + result = append(result, NewXmlTreeImporterElement(c)) } return result } -func (x *XmlTreeImporter) GetKeyValue() (string, error) { +func (x *XmlTreeImporterElement) GetKeyValue() (string, error) { return x.elem.Text(), nil } -func (x *XmlTreeImporter) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { +func (x *XmlTreeImporterElement) GetTVValue(ctx context.Context, slt *sdcpb.SchemaLeafType) (*sdcpb.TypedValue, error) { return utils.Convert(ctx, x.elem.Text(), slt) } -func (x *XmlTreeImporter) GetName() string { +func (x *XmlTreeImporterElement) GetName() string { return x.elem.Tag } // Function to ensure JsonTreeImporter implements ImportConfigAdapter (optional) var _ importer.ImportConfigAdapter = (*XmlTreeImporter)(nil) +var _ importer.ImportConfigAdapterElement = (*XmlTreeImporterElement)(nil) diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index d22720ec..d7a4106f 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -92,7 +92,7 @@ func TestXmlTreeImporter(t *testing.T) { t.Fatal(err) } - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element), "owner1", 5, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/leaf_entry.go b/pkg/tree/leaf_entry.go index 30856fb1..34f7771d 100644 --- a/pkg/tree/leaf_entry.go +++ b/pkg/tree/leaf_entry.go @@ -69,6 +69,8 @@ func (l *LeafEntry) MarkNew() { } func (l *LeafEntry) RemoveDeleteFlag() *LeafEntry { + l.mu.Lock() + defer l.mu.Unlock() l.Delete = false return l } @@ -137,9 +139,15 @@ func (l *LeafEntry) MarkExpliciteDelete() { l.IsNew = false } +func (l *LeafEntry) NonRevertive() bool { + l.mu.RLock() + defer l.mu.RUnlock() + return l.parentEntry.getTreeContext().IsNonRevertiveIntent(l.Owner()) +} + // String returns a string representation of the LeafEntry func (l *LeafEntry) String() string { - return fmt.Sprintf("Owner: %s, Priority: %d, Value: %s, New: %t, Delete: %t, Update: %t, DeleteIntendedOnly: %t, ExplicitDelete: %t", l.Owner(), l.Priority(), l.Value().ToString(), l.GetNewFlag(), l.GetDeleteFlag(), l.GetUpdateFlag(), l.GetDeleteOnlyIntendedFlag(), l.GetExplicitDeleteFlag()) + return fmt.Sprintf("Owner: %s, Priority: %d, Value: %s, New: %t, Delete: %t, Update: %t, DeleteIntendedOnly: %t, ExplicitDelete: %t, Non-Revertive: %t", l.Owner(), l.Priority(), l.Value().ToString(), l.GetNewFlag(), l.GetDeleteFlag(), l.GetUpdateFlag(), l.GetDeleteOnlyIntendedFlag(), l.GetExplicitDeleteFlag(), l.NonRevertive()) } // Compare used for slices.SortFunc. Sorts by path and if equal paths then by owner as the second criteria diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index db9806a3..22f57767 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -311,6 +311,8 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau // figure out the highest precedence LeafEntry var highest *LeafEntry + // the second highests is the backup in case the highest is marked for deletion + // so this is not actually the second highest always, but the next candidate var secondHighest *LeafEntry for _, e := range lv.les { // first entry set result to it @@ -327,13 +329,13 @@ func (lv *LeafVariants) GetHighestPrecedence(onlyNewOrUpdated bool, includeDefau highest = e } else { // check if the update is at least higher prio (lower number) then the secondHighest - if secondHighest == nil || secondHighest.Priority() > e.Priority() { + if secondHighest == nil || secondHighest.Priority() > e.Priority() && !e.GetDeleteFlag() { secondHighest = e } } } - if highest.IsExplicitDelete && !includeExplicitDelete { + if highest == nil || highest.IsExplicitDelete && !includeExplicitDelete { return nil } @@ -373,7 +375,7 @@ func (lv *LeafVariants) highestIsUnequalRunning(highest *LeafEntry) bool { } // if highest is not new or updated and highest is non-revertive - if !highest.IsNew && !highest.IsUpdated && lv.tc.nonRevertiveInfo[highest.Update.Owner()] { + if !highest.IsNew && !highest.IsUpdated && lv.tc.IsNonRevertiveIntent(highest.Update.Owner()) { return false } diff --git a/pkg/tree/parallelImporter.go b/pkg/tree/parallelImporter.go index 83f7ee2d..f7c03e22 100644 --- a/pkg/tree/parallelImporter.go +++ b/pkg/tree/parallelImporter.go @@ -25,9 +25,7 @@ type importTask struct { func (s *sharedEntryAttributes) ImportConfig( ctx context.Context, - importerElement importer.ImportConfigAdapterElement, - intentName string, - intentPrio int32, + importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags, ) error { p := pool.NewWorkerPool[importTask](ctx, 1) @@ -35,7 +33,7 @@ func (s *sharedEntryAttributes) ImportConfig( p.Start(importHandler) // seed root - if err := p.Submit(importTask{entry: s, importerElement: importerElement, intentName: intentName, intentPrio: intentPrio, insertFlags: insertFlags, treeContext: s.treeContext, leafListLock: &sync.Map{}}); err != nil { + if err := p.Submit(importTask{entry: s, importerElement: importer, intentName: importer.GetName(), intentPrio: importer.GetPriority(), insertFlags: insertFlags, treeContext: s.treeContext, leafListLock: &sync.Map{}}); err != nil { return err } diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 53341d59..400551a0 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -88,23 +88,30 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { - r.treeContext.SetActualOwner(intentName) +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error { + r.treeContext.SetActualOwner(importer.GetName()) e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) if err != nil { return err } + // store non revertive info + r.treeContext.nonRevertiveInfo[importer.GetName()] = importer.GetNonRevertive() - r.explicitDeletes.Add(intentName, intentPrio, importer.GetDeletes()) + // store explicit deletes + r.explicitDeletes.Add(importer.GetName(), importer.GetPriority(), importer.GetDeletes()) - return e.ImportConfig(ctx, importer, intentName, intentPrio, flags) + return e.ImportConfig(ctx, importer, flags) } func (r *RootEntry) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { r.explicitDeletes.Add(intentName, priority, pathset) } +func (r *RootEntry) SetNonRevertiveIntent(intentName string, nonRevertive bool) { + r.treeContext.nonRevertiveInfo[intentName] = nonRevertive +} + func (r *RootEntry) Validate(ctx context.Context, vCfg *config.Validation, taskpoolFactory pool.VirtualPoolFactory) (types.ValidationResults, *types.ValidationStats) { // perform validation // we use a channel and cumulate all the errors @@ -184,7 +191,7 @@ func (r *RootEntry) GetDeviations(ctx context.Context, ch chan<- *types.Deviatio r.sharedEntryAttributes.GetDeviations(ctx, ch, true) } -func (r *RootEntry) TreeExport(owner string, priority int32, nonRevertive bool) (*tree_persist.Intent, error) { +func (r *RootEntry) TreeExport(owner string, priority int32) (*tree_persist.Intent, error) { treeExport, err := r.sharedEntryAttributes.TreeExport(owner) if err != nil { return nil, err @@ -202,7 +209,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32, nonRevertive bool) IntentName: owner, Root: rootExportEntry, Priority: priority, - NonRevertive: nonRevertive, + NonRevertive: r.treeContext.nonRevertiveInfo[owner], ExplicitDeletes: explicitDeletes, }, nil } diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index af432937..4e23d0fa 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -303,7 +303,7 @@ func TestRootEntry_TreeExport(t *testing.T) { sharedEntryAttributes: tt.sharedEntryAttributes(), explicitDeletes: NewDeletePaths(), } - got, err := r.TreeExport(tt.args.owner, tt.args.priority, false) + got, err := r.TreeExport(tt.args.owner, tt.args.priority) if (err != nil) != tt.wantErr { t.Fatalf("RootEntry.TreeExport() error = %v, wantErr %v", err, tt.wantErr) return @@ -516,7 +516,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } - err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny), "owner1", 5, types.NewUpdateInsertFlags()) + err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags()) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sharedEntryAttributes.go b/pkg/tree/sharedEntryAttributes.go index d8025333..60bc58a1 100644 --- a/pkg/tree/sharedEntryAttributes.go +++ b/pkg/tree/sharedEntryAttributes.go @@ -121,6 +121,10 @@ func (s *sharedEntryAttributes) GetRoot() Entry { return s.parent.GetRoot() } +func (s sharedEntryAttributes) getTreeContext() *TreeContext { + return s.treeContext +} + // loadDefaults helper to populate defaults on the initializiation of the sharedEntryAttribute func (s *sharedEntryAttributes) loadDefaults(ctx context.Context) error { diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index ea1f8187..539690f7 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -146,7 +146,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Error(err) } @@ -654,7 +654,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Fatal(err) } @@ -776,7 +776,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Fatal(err) } @@ -1110,7 +1110,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println(root.String()) - treepersist, err := root.TreeExport(owner1, owner1Prio, false) + treepersist, err := root.TreeExport(owner1, owner1Prio) if err != nil { t.Error(err) return @@ -1130,7 +1130,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), owner1, owner1Prio, false, flagsExisting) + err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting) if err != nil { t.Error(err) return @@ -1282,7 +1282,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Fatal(err) } @@ -1451,7 +1451,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 7d9b4722..6091d1e9 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -236,7 +236,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner1, 500, false, newFlag) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 19b8c95f..bed5287e 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -65,9 +65,9 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } @@ -179,10 +179,10 @@ func TestValidate_RangesSigned(t *testing.T) { } // new json tree importer - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } @@ -313,10 +313,10 @@ func TestValidate_RangesUnSigned(t *testing.T) { } // new json tree importer - jimporter := json_importer.NewJsonTreeImporter(jsonConfig) + jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, "owner1", 5, false, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) if err != nil { t.Error(err) } diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index a9fa5cec..7ff7ea43 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -138,7 +138,7 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem } type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, intentName string, intentPrio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error } func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { @@ -156,7 +156,7 @@ func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root Root return err } - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny), owner, prio, nonRevertive, flags) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) if err != nil { return err } From e2dbf5f88750506114b26a71da8d0fae440e395b Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 27 Jan 2026 12:02:45 +0100 Subject: [PATCH 08/17] added mock gen for cacheClientBound --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 261defd0..dae01961 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ mocks-gen: mocks-rm ## Generate mocks for all the defined interfaces. mockgen -package=mockschema -source=pkg/schema/schema_client.go -destination=$(MOCKDIR)/mockschema/client.go mockgen -package=mockschemaclientbound -source=pkg/datastore/clients/schema/schemaClientBound.go -destination=$(MOCKDIR)/mockschemaclientbound/client.go mockgen -package=mockcacheclient -source=pkg/cache/cache.go -destination=$(MOCKDIR)/mockcacheclient/client.go + mockgen -package=mockcacheclient -source=pkg/cache/cacheClientBound.go -destination=$(MOCKDIR)/mockcacheclient/clientbound.go mockgen -package=mocktarget -source=pkg/datastore/target/target.go -destination=$(MOCKDIR)/mocktarget/target.go mockgen -package=mockTreeEntry -source=pkg/tree/entry.go -destination=$(MOCKDIR)/mocktreeentry/entry.go From 7e311ed7d68f2794fc6ee613ee1b4758bb70e700 Mon Sep 17 00:00:00 2001 From: steiler Date: Wed, 28 Jan 2026 16:13:56 +0100 Subject: [PATCH 09/17] update --- mocks/mocktreeentry/entry.go | 9 +- pkg/datastore/intent_rpc.go | 3 +- pkg/datastore/sync.go | 4 +- pkg/datastore/sync_test.go | 12 ++- pkg/datastore/transaction_rpc.go | 4 +- pkg/datastore/tree_operation_test.go | 7 +- .../tree_operation_validation_test.go | 3 +- pkg/pool/virtual_pool.go | 5 +- pkg/tree/entry.go | 3 +- pkg/tree/entry_map.go | 11 ++ .../proto/proto_tree_importer_test.go | 9 +- .../importer/xml/xml_tree_importer_test.go | 10 +- pkg/tree/leaf_entry.go | 4 + pkg/tree/parallelImporter.go | 102 +++++++++++++++--- pkg/tree/processor_blame_config.go | 60 +++-------- pkg/tree/processor_remove_deleted.go | 5 +- pkg/tree/root_entry.go | 4 +- pkg/tree/root_entry_test.go | 9 +- pkg/tree/sharedEntryAttributes_test.go | 18 ++-- pkg/tree/validation_entry_leafref_test.go | 5 +- pkg/tree/validation_range_test.go | 19 ++-- pkg/utils/testhelper/utils.go | 7 +- 22 files changed, 207 insertions(+), 106 deletions(-) diff --git a/mocks/mocktreeentry/entry.go b/mocks/mocktreeentry/entry.go index ccc82fc1..1ee421e0 100644 --- a/mocks/mocktreeentry/entry.go +++ b/mocks/mocktreeentry/entry.go @@ -15,6 +15,7 @@ import ( etree "github.com/beevik/etree" config "github.com/sdcio/data-server/pkg/config" + pool "github.com/sdcio/data-server/pkg/pool" tree "github.com/sdcio/data-server/pkg/tree" importer "github.com/sdcio/data-server/pkg/tree/importer" types "github.com/sdcio/data-server/pkg/tree/types" @@ -362,17 +363,17 @@ func (mr *MockEntryMockRecorder) HoldsLeafvariants() *gomock.Call { } // ImportConfig mocks base method. -func (m *MockEntry) ImportConfig(ctx context.Context, arg1 importer.ImportConfigAdapterElement, intentName string, intentPrio int32, flags *types.UpdateInsertFlags) error { +func (m *MockEntry) ImportConfig(ctx context.Context, arg1 importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ImportConfig", ctx, arg1, intentName, intentPrio, flags) + ret := m.ctrl.Call(m, "ImportConfig", ctx, arg1, flags, pool) ret0, _ := ret[0].(error) return ret0 } // ImportConfig indicates an expected call of ImportConfig. -func (mr *MockEntryMockRecorder) ImportConfig(ctx, arg1, intentName, intentPrio, flags any) *gomock.Call { +func (mr *MockEntryMockRecorder) ImportConfig(ctx, arg1, flags, pool any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportConfig", reflect.TypeOf((*MockEntry)(nil).ImportConfig), ctx, arg1, intentName, intentPrio, flags) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportConfig", reflect.TypeOf((*MockEntry)(nil).ImportConfig), ctx, arg1, flags, pool) } // IsRoot mocks base method. diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index e89bfea2..37b309fa 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -22,6 +22,7 @@ import ( "github.com/beevik/etree" targettypes "github.com/sdcio/data-server/pkg/datastore/target/types" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer/proto" "github.com/sdcio/data-server/pkg/tree/types" @@ -77,7 +78,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index c84f5f7e..d6aeba75 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -49,7 +49,8 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i // import new config if provided if importer != nil { - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags()) + importPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) + err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), importPool) if err != nil { return err } @@ -165,6 +166,7 @@ func (d *Datastore) performRevert(ctx context.Context, t *tree.RootEntry) error } if performApply { + log.Info("reverting after sync") resp, err := d.applyIntent(ctx, t) if err != nil { respJ := protojson.MarshalOptions{Multiline: false} diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index e45e41b5..953ca368 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -80,7 +80,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -185,7 +186,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -288,7 +290,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -396,7 +399,8 @@ func TestApplyToRunning(t *testing.T) { d := tt.resultFunc() - err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags()) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatalf("failed to import test config: %v", err) } diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 2f6ccef6..7f58a7e0 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -86,7 +86,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags()) + err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) if err != nil { return nil, err } diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index d31915b8..353061c7 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1519,7 +1519,11 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + + err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, importPool) + if err != nil { t.Error(err) } @@ -1531,7 +1535,6 @@ func TestDatastore_populateTree(t *testing.T) { } fmt.Println(root.String()) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) fmt.Printf("Validation Errors:\n%v\n", strings.Join(validationResult.ErrorsStr(), "\n")) diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index bda39bb8..be213a55 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -198,7 +198,8 @@ func TestDatastore_validateTree(t *testing.T) { importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) - err = root.ImportConfig(ctx, path, importer, flagsNew) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, path, importer, flagsNew, vp) if err != nil { t.Error(err) } diff --git a/pkg/pool/virtual_pool.go b/pkg/pool/virtual_pool.go index 759c8172..0ef4b843 100644 --- a/pkg/pool/virtual_pool.go +++ b/pkg/pool/virtual_pool.go @@ -318,9 +318,12 @@ func (v *VirtualPool) FirstError() error { } // Errors returns a snapshot of collected errors for tolerant virtual pools. -// For fail-fast virtual pools this returns nil. +// For fail-fast virtual pools this returns the first error if any. func (v *VirtualPool) Errors() []error { if v.ec == nil { + if err := v.FirstError(); err != nil { + return []error{err} + } return nil } return v.ec.Errors() diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 94725b7e..9c5c37fd 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -6,6 +6,7 @@ import ( "github.com/beevik/etree" "github.com/sdcio/data-server/pkg/config" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/sdc-protos/tree_persist" @@ -131,7 +132,7 @@ type Entry interface { ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error + ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) diff --git a/pkg/tree/entry_map.go b/pkg/tree/entry_map.go index ecdccad8..cd94aa28 100644 --- a/pkg/tree/entry_map.go +++ b/pkg/tree/entry_map.go @@ -1,3 +1,14 @@ package tree +import "sort" + type EntryMap map[string]Entry + +func (e EntryMap) SortedKeys() []string { + keys := make([]string, 0, len(e)) + for k := range e { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 172aed09..98c4325f 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/google/go-cmp/cmp" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" jimport "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" @@ -127,7 +129,9 @@ func TestProtoTreeImporter(t *testing.T) { } jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) - err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags()) + + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatal(err) } @@ -153,7 +157,8 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags()) + vp2 := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vp2) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index d7a4106f..a4c629cd 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -3,10 +3,12 @@ package xml import ( "context" "fmt" + "runtime" "testing" "github.com/beevik/etree" "github.com/google/go-cmp/cmp" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" @@ -92,7 +94,13 @@ func TestXmlTreeImporter(t *testing.T) { t.Fatal(err) } - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + + err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), importPool) + sharedPool.CloseForSubmit() + sharedPool.Wait() + if err != nil { t.Fatal(err) } diff --git a/pkg/tree/leaf_entry.go b/pkg/tree/leaf_entry.go index 34f7771d..fef0c61c 100644 --- a/pkg/tree/leaf_entry.go +++ b/pkg/tree/leaf_entry.go @@ -142,6 +142,10 @@ func (l *LeafEntry) MarkExpliciteDelete() { func (l *LeafEntry) NonRevertive() bool { l.mu.RLock() defer l.mu.RUnlock() + // this is a hack that makes the tests pass + if l.parentEntry == nil { + return false + } return l.parentEntry.getTreeContext().IsNonRevertiveIntent(l.Owner()) } diff --git a/pkg/tree/parallelImporter.go b/pkg/tree/parallelImporter.go index f7c03e22..9894c12f 100644 --- a/pkg/tree/parallelImporter.go +++ b/pkg/tree/parallelImporter.go @@ -5,6 +5,7 @@ import ( "fmt" "slices" "sync" + "sync/atomic" "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/importer" @@ -13,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) -type importTask struct { +type importConfigTask struct { entry Entry importerElement importer.ImportConfigAdapterElement intentName string @@ -23,28 +24,57 @@ type importTask struct { leafListLock *sync.Map } -func (s *sharedEntryAttributes) ImportConfig( - ctx context.Context, - importer importer.ImportConfigAdapter, - insertFlags *types.UpdateInsertFlags, -) error { - p := pool.NewWorkerPool[importTask](ctx, 1) +type ImportConfigProcessor struct { + importer importer.ImportConfigAdapter + insertFlags *types.UpdateInsertFlags +} - p.Start(importHandler) +func NewImportConfigProcessor(importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { + return &ImportConfigProcessor{ + importer: importer, + insertFlags: insertFlags, + } +} - // seed root - if err := p.Submit(importTask{entry: s, importerElement: importer, intentName: importer.GetName(), intentPrio: importer.GetPriority(), insertFlags: insertFlags, treeContext: s.treeContext, leafListLock: &sync.Map{}}); err != nil { +func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, workerPool pool.VirtualPoolI) error { + + t := importConfigTask{ + entry: e, + importerElement: p.importer, + intentName: p.importer.GetName(), + intentPrio: p.importer.GetPriority(), + insertFlags: p.insertFlags, + treeContext: e.getTreeContext(), + leafListLock: &sync.Map{}, + } + + if err := workerPool.Submit(t); err != nil { + workerPool.CloseAndWait() return err } - // signal we are done seeding external tasks (workers may still submit) - p.CloseForSubmit() + workerPool.CloseAndWait() + + if err := workerPool.FirstError(); err != nil { + return err + } + // TODO: support tolerant mode? Processor usually decides what to return based on pool mode, + // but FirstError() works for fail-fast. For tolerant, we might want Errors(). + // But ImportConfig usually stops on error? + return nil +} - // wait for the import to finish (or error) - return p.Wait() +func (s *sharedEntryAttributes) ImportConfig( + ctx context.Context, + importer importer.ImportConfigAdapter, + insertFlags *types.UpdateInsertFlags, + workerPool pool.VirtualPoolI, +) error { + processor := NewImportConfigProcessor(importer, insertFlags) + return processor.Run(ctx, s, workerPool) } -func importHandler(ctx context.Context, task importTask, submit func(importTask) error) error { +func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) error) error { elem := task.entry.PathName() _ = elem @@ -78,7 +108,7 @@ func importHandler(ctx context.Context, task importTask, submit func(importTask) } // submit resolved entry with same adapter element // return importHandler(ctx, importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext}, submit) - return submit(importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}) + return submit(importConfigTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}) } // presence container or children @@ -103,7 +133,7 @@ func importHandler(ctx context.Context, task importTask, submit func(importTask) return fmt.Errorf("error inserting %s at %s: %w", childElt.GetName(), task.entry.SdcpbPath().ToXPath(false), err) } } - if err := submit(importTask{entry: child, importerElement: childElt, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}); err != nil { + if err := submit(importConfigTask{entry: child, importerElement: childElt, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}); err != nil { return err } } @@ -169,3 +199,41 @@ func importHandler(ctx context.Context, task importTask, submit func(importTask) return nil } } + +type ImportStat struct { + newEntries atomic.Int64 + updatedEntries atomic.Int64 +} + +func NewImportStat() *ImportStat { + return &ImportStat{} +} + +func (is *ImportStat) String() string { + return fmt.Sprintf("NewEntries: %d, UpdatedEntries: %d", is.newEntries.Load(), is.updatedEntries.Load()) +} + +func (is *ImportStat) Join(i *ImportStat) { + is.newEntries.Add(i.newEntries.Load()) + is.updatedEntries.Add(i.updatedEntries.Load()) +} + +func (is *ImportStat) AddNewEntryStat() { + is.newEntries.Add(1) +} + +func (is *ImportStat) AddUpdatedEntryStat() { + is.updatedEntries.Add(1) +} + +func (is *ImportStat) GetNewEntries() int64 { + return is.newEntries.Load() +} + +func (is *ImportStat) GetUpdatedEntries() int64 { + return is.updatedEntries.Load() +} + +func (is *ImportStat) Changed() bool { + return is.newEntries.Load() > 0 || is.updatedEntries.Load() > 0 +} diff --git a/pkg/tree/processor_blame_config.go b/pkg/tree/processor_blame_config.go index 8c6e227e..9c42c918 100644 --- a/pkg/tree/processor_blame_config.go +++ b/pkg/tree/processor_blame_config.go @@ -3,7 +3,6 @@ package tree import ( "context" "errors" - "sync" "github.com/sdcio/data-server/pkg/pool" sdcpb "github.com/sdcio/sdc-protos/sdcpb" @@ -35,34 +34,17 @@ func NewBlameConfigProcessorConfig(includeDefaults bool) *BlameConfigProcessorCo // VirtualFailFast to stop on first error. // Returns the blame tree structure and any error encountered. func (p *BlameConfigProcessor) Run(ctx context.Context, e Entry, pool pool.VirtualPoolI) (*sdcpb.BlameTreeElement, error) { - dropChan := make(chan *DropBlameChild, 10) - - wg := &sync.WaitGroup{} - wg.Add(1) - // Execute the deletes in a separate goroutine - go func(dC <-chan *DropBlameChild) { - for elem := range dC { - elem.Exec() - } - wg.Done() - }(dropChan) - blameTask := NewBlameConfigTask(e, dropChan, p.config) + blameTask := NewBlameConfigTask(e, p.config) if err := pool.Submit(blameTask); err != nil { - // Clean up pool and channels even on early error + // Clean up pool even on early error pool.CloseAndWait() - close(dropChan) - wg.Wait() return nil, err } // Close pool and wait for all tasks to complete before checking errors pool.CloseAndWait() - // Close the dropChan channel and wait for cleanup goroutine - close(dropChan) - wg.Wait() - // Return first error for fail-fast mode, or combined errors for tolerant mode if errs := pool.Errors(); len(errs) > 0 { return blameTask.self, errors.Join(errs...) @@ -75,16 +57,14 @@ type BlameConfigTask struct { parent *sdcpb.BlameTreeElement self *sdcpb.BlameTreeElement selfEntry Entry - dropChan chan<- *DropBlameChild } -func NewBlameConfigTask(e Entry, dropChan chan<- *DropBlameChild, c *BlameConfigProcessorConfig) *BlameConfigTask { +func NewBlameConfigTask(e Entry, c *BlameConfigProcessorConfig) *BlameConfigTask { return &BlameConfigTask{ config: c, parent: nil, self: &sdcpb.BlameTreeElement{}, selfEntry: e, - dropChan: dropChan, } } @@ -111,13 +91,19 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) t.self.SetDeviationValue(runningLe.Value()) } } - } else { - // if it is default but no default is meant to be returned - t.dropChan <- &DropBlameChild{parent: t.parent, dropElem: t.self} } } - for _, childEntry := range t.selfEntry.GetChilds(DescendMethodActiveChilds) { + childs := t.selfEntry.GetChilds(DescendMethodActiveChilds) + for _, childKey := range childs.SortedKeys() { + childEntry := childs[childKey] + childHighestLe := childEntry.GetLeafVariantEntries().GetHighestPrecedence(false, true, true) + if childHighestLe != nil { + if childHighestLe.Update.Owner() == DefaultsIntentName && !t.config.includeDefaults { + continue + } + } + child := &sdcpb.BlameTreeElement{Name: childEntry.PathName()} t.self.AddChild(child) @@ -127,7 +113,6 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) parent: t.self, self: child, selfEntry: childEntry, - dropChan: t.dropChan, } // Submit may fail if pool is closed or fail-fast error occurred if err := submit(task); err != nil { @@ -137,22 +122,3 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) return nil } - -type DropBlameChild struct { - parent *sdcpb.BlameTreeElement - dropElem *sdcpb.BlameTreeElement -} - -func (d *DropBlameChild) Exec() { - // from parent drop the child dropElem - index := -1 - for i, child := range d.parent.GetChilds() { - if child == d.dropElem { - index = i - break - } - } - if index != -1 { - d.parent.Childs = append(d.parent.Childs[:index], d.parent.Childs[index+1:]...) - } -} diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go index f1715a45..26932160 100644 --- a/pkg/tree/processor_remove_deleted.go +++ b/pkg/tree/processor_remove_deleted.go @@ -59,10 +59,7 @@ func (p *RemoveDeletedProcessor) Run(e Entry, pool pool.VirtualPoolI) error { pool.CloseAndWait() // Return first error for fail-fast mode, or combined errors for tolerant mode - if errs := pool.Errors(); len(errs) > 0 { - return errors.Join(errs...) - } - return pool.FirstError() + return errors.Join(pool.Errors()...) } type removeDeletedTask struct { diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 400551a0..de2c80e0 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -88,7 +88,7 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error { +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error { r.treeContext.SetActualOwner(importer.GetName()) e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) @@ -101,7 +101,7 @@ func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, impo // store explicit deletes r.explicitDeletes.Add(importer.GetName(), importer.GetPriority(), importer.GetDeletes()) - return e.ImportConfig(ctx, importer, flags) + return e.ImportConfig(ctx, importer, flags, pool) } func (r *RootEntry) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 4e23d0fa..7a69ffef 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "runtime" "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -47,6 +49,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -95,6 +98,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -161,6 +165,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -261,6 +266,7 @@ func TestRootEntry_TreeExport(t *testing.T) { childsMutex: sync.RWMutex{}, schemaMutex: sync.RWMutex{}, cacheMutex: sync.Mutex{}, + treeContext: tc, } result.leafVariants = newLeafVariants(tc, result) @@ -516,7 +522,8 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } - err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags()) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags(), vp) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index 539690f7..fa7501fd 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -146,7 +146,8 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Error(err) } @@ -654,7 +655,8 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Fatal(err) } @@ -776,7 +778,8 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Fatal(err) } @@ -1130,7 +1133,8 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vp) if err != nil { t.Error(err) return @@ -1282,7 +1286,8 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Fatal(err) } @@ -1451,7 +1456,8 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 6091d1e9..3a32d32f 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -236,7 +238,8 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index bed5287e..2ce12413 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -67,7 +67,9 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) if err != nil { t.Error(err) } @@ -78,7 +80,6 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { } valConf := validationConfig.DeepCopy() - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, valConf, sharedPool) @@ -181,8 +182,12 @@ func TestValidate_RangesSigned(t *testing.T) { // new json tree importer jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) + if err != nil { t.Error(err) } @@ -192,7 +197,6 @@ func TestValidate_RangesSigned(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) t.Logf("Validation Errors:\n%s", strings.Join(validationResult.ErrorsStr(), "\n")) @@ -315,8 +319,12 @@ func TestValidate_RangesUnSigned(t *testing.T) { // new json tree importer jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) + // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) + if err != nil { t.Error(err) } @@ -327,7 +335,6 @@ func TestValidate_RangesUnSigned(t *testing.T) { } // run validation - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) validationResults, _ := root.Validate(ctx, validationConfig, sharedPool) t.Logf("Validation Errors:\n%s", strings.Join(validationResults.ErrorsStr(), "\n")) diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 7ff7ea43..498a90c8 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "runtime" "slices" "strings" "testing" @@ -11,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" "github.com/sdcio/data-server/mocks/mockschemaclientbound" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/importer" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" @@ -138,7 +140,7 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem } type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags) error + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error } func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { @@ -156,7 +158,8 @@ func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root Root return err } - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) + vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vp) if err != nil { return err } From 99ade8039eebddda9878afb2c2077a7477aa53f7 Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 29 Jan 2026 10:25:59 +0100 Subject: [PATCH 10/17] change the ImportConfig signature to require pool.VirtualPoolFactory --- pkg/datastore/intent_rpc.go | 3 +- pkg/datastore/sync.go | 37 +----- pkg/datastore/sync_test.go | 26 ++--- pkg/datastore/transaction_rpc.go | 8 +- pkg/datastore/tree_operation_test.go | 7 +- .../tree_operation_validation_test.go | 4 +- pkg/tree/entry.go | 3 +- pkg/tree/entry_test.go | 9 +- .../proto/proto_tree_importer_test.go | 8 +- .../importer/xml/xml_tree_importer_test.go | 3 +- pkg/tree/leaf_variants.go | 9 +- ...allelImporter.go => processor_importer.go} | 109 +++++++++++------- pkg/tree/processor_mark_owner_delete.go | 3 +- pkg/tree/processor_remove_deleted.go | 6 +- pkg/tree/processor_reset_flags.go | 5 +- pkg/tree/processor_reset_flags_test.go | 3 +- pkg/tree/root_entry.go | 4 +- pkg/tree/root_entry_test.go | 5 +- pkg/tree/sharedEntryAttributes.go | 9 +- pkg/tree/sharedEntryAttributes_test.go | 27 +++-- pkg/tree/validation_entry_leafref_test.go | 4 +- pkg/tree/validation_range_test.go | 10 +- pkg/tree/xml_test.go | 3 +- pkg/utils/testhelper/utils.go | 6 +- 24 files changed, 152 insertions(+), 159 deletions(-) rename pkg/tree/{parallelImporter.go => processor_importer.go} (70%) diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 37b309fa..4d740f3a 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -22,7 +22,6 @@ import ( "github.com/beevik/etree" targettypes "github.com/sdcio/data-server/pkg/datastore/target/types" - "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer/proto" "github.com/sdcio/data-server/pkg/tree/types" @@ -78,7 +77,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) + err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index d6aeba75..8a411d4f 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -2,10 +2,8 @@ package datastore import ( "context" - "errors" "sync" - "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" treetypes "github.com/sdcio/data-server/pkg/tree/types" @@ -24,7 +22,6 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i defer syncTreeUnlock() // create a virtual task pool for delete operations - deleteMarkerPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) for _, delete := range deletes { // navigate to delete path deleteRoot, err := d.syncTree.NavigateSdcpbPath(ctx, delete) @@ -33,42 +30,24 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i continue } // apply delete marker, setting owner delete flag on running intent - err = tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tree.RunningIntentName, false)).Run(deleteRoot, deleteMarkerPool) + err = tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tree.RunningIntentName, false)).Run(deleteRoot, d.taskPool) if err != nil { log.Error(err, "failed applying delete to path", "path", delete.ToXPath(false)) continue } } - // close the delete marker pool for submission and wait - deleteMarkerPool.CloseAndWait() - err := deleteMarkerPool.FirstError() - if err != nil { - return err - } - // import new config if provided if importer != nil { - importPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), importPool) + err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return err } } - // create a virtual task pool for removeDeleted operations - removeDeletedPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) - // run remove deleted processor to clean up entries marked as deleted by owner delProcessorParams := tree.NewRemoveDeletedProcessorParameters(tree.RunningIntentName) - err = tree.NewRemoveDeletedProcessor(delProcessorParams).Run(d.syncTree.GetRoot(), removeDeletedPool) - if err != nil { - return err - } - - // close the remove deleted pool for submission and wait - removeDeletedPool.CloseAndWait() - err = errors.Join(removeDeletedPool.Errors()...) + err := tree.NewRemoveDeletedProcessor(delProcessorParams).Run(d.syncTree.GetRoot(), d.taskPool) if err != nil { return err } @@ -93,19 +72,13 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } // run reset flags processor to reset flags - resetFlagsPool := d.taskPool.NewVirtualPool(pool.VirtualTolerant) resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(true, true, true) - err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(d.syncTree.GetRoot(), resetFlagsPool) + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(d.syncTree.GetRoot(), d.taskPool) if err != nil { return err } - // close the resetFlags pool for submission and wait - resetFlagsPool.CloseAndWait() - if errors.Join(resetFlagsPool.Errors()...) != nil { - return err - } - + // create a deep copy of the sync tree for revert operation syncTreeCopy, err := d.syncTree.DeepCopy(ctx) if err != nil { return err diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index 953ca368..abd5c84c 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -80,8 +80,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -186,8 +186,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -290,8 +290,8 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -399,8 +399,8 @@ func TestApplyToRunning(t *testing.T) { d := tt.resultFunc() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -412,20 +412,12 @@ func TestApplyToRunning(t *testing.T) { fmt.Println(syncTree.String()) - vpool := datastore.taskPool.NewVirtualPool(pool.VirtualFailFast) - resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) - err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), vpool) + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), datastore.taskPool) if err != nil { t.Fatalf("failed to reset flags: %v", err) } - vpool.CloseAndWait() - err = vpool.FirstError() - if err != nil { - t.Fatalf("failed to run reset flags processor: %v", err) - } - fmt.Println("Adjusted flags count:", resetFlagsProcessorParams.GetAdjustedFlagsCount()) if diff := cmp.Diff(resultRoot.String(), syncTree.String()); diff != "" { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 7f58a7e0..21a8fe76 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -9,7 +9,6 @@ import ( "time" "github.com/sdcio/data-server/pkg/datastore/types" - "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree" treeproto "github.com/sdcio/data-server/pkg/tree/importer/proto" treetypes "github.com/sdcio/data-server/pkg/tree/types" @@ -86,7 +85,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) + err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -179,7 +178,7 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool.NewVirtualPool(pool.VirtualFailFast)) + err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -219,10 +218,9 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ oldIntentContent := lvs.ToPathAndUpdateSlice() - deleteVisitorPool := d.taskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(intent.GetName(), intent.GetOnlyIntended())) - err := ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err := ownerDeleteMarker.Run(root.GetRoot(), d.taskPool) if err != nil { return nil, err } diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 353061c7..9fd1efa5 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1502,10 +1502,8 @@ func TestDatastore_populateTree(t *testing.T) { } sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tt.intentName, false)) - - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1520,9 +1518,8 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, importPool) + err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, sharedPool) if err != nil { t.Error(err) diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index be213a55..61a2ada4 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -198,8 +198,8 @@ func TestDatastore_validateTree(t *testing.T) { importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, path, importer, flagsNew, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, path, importer, flagsNew, vpf) if err != nil { t.Error(err) } diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 9c5c37fd..57eff845 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -132,7 +132,7 @@ type Entry interface { ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error + ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) @@ -173,6 +173,7 @@ type LeafVariantEntries interface { GetByOwner(owner string) *LeafEntry RemoveDeletedByOwner(owner string) *LeafEntry Add(l *LeafEntry) + AddWithStats(l *LeafEntry, stats *ImportStats) Length() int } diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index beab79a7..b57b9c2e 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -567,10 +567,9 @@ func Test_Entry_Three(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -842,10 +841,9 @@ func Test_Entry_Four(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1217,10 +1215,9 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 98c4325f..14750a7c 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -130,8 +130,8 @@ func TestProtoTreeImporter(t *testing.T) { jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatal(err) } @@ -157,8 +157,8 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - vp2 := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vp2) + vpf2 := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vpf2) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index a4c629cd..dc1ad823 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -95,9 +95,8 @@ func TestXmlTreeImporter(t *testing.T) { } sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), importPool) + err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), sharedPool) sharedPool.CloseForSubmit() sharedPool.Wait() diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index 22f57767..b0709aaa 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -25,7 +25,7 @@ func newLeafVariants(tc *TreeContext, parentEnty Entry) *LeafVariants { } } -func (lv *LeafVariants) Add(le *LeafEntry) { +func (lv *LeafVariants) AddWithStats(le *LeafEntry, stats *ImportStats) { if leafVariant := lv.GetByOwner(le.Owner()); leafVariant != nil { if leafVariant.Update.Equal(le.Update) { // it seems like the element was not deleted, so drop the delete flag @@ -33,15 +33,22 @@ func (lv *LeafVariants) Add(le *LeafEntry) { } else { // if a leafentry of the same owner exists with different value, mark it for update leafVariant.MarkUpdate(le.Update) + stats.IncrementUpdated() + } } else { lv.lesMutex.Lock() defer lv.lesMutex.Unlock() // if LeafVaraint with same owner does not exist, add the new entry lv.les = append(lv.les, le) + stats.IncrementNew() } } +func (lv *LeafVariants) Add(le *LeafEntry) { + lv.AddWithStats(le, nil) +} + // Items iterator for the LeafVariants func (lv *LeafVariants) Items() iter.Seq[*LeafEntry] { return func(yield func(*LeafEntry) bool) { diff --git a/pkg/tree/parallelImporter.go b/pkg/tree/processor_importer.go similarity index 70% rename from pkg/tree/parallelImporter.go rename to pkg/tree/processor_importer.go index 9894c12f..281ee9d2 100644 --- a/pkg/tree/parallelImporter.go +++ b/pkg/tree/processor_importer.go @@ -17,35 +17,62 @@ import ( type importConfigTask struct { entry Entry importerElement importer.ImportConfigAdapterElement - intentName string - intentPrio int32 - insertFlags *types.UpdateInsertFlags - treeContext *TreeContext - leafListLock *sync.Map + params *ImportConfigProcessorParams +} + +type ImportConfigProcessorParams struct { + intentName string + intentPrio int32 + insertFlags *types.UpdateInsertFlags + treeContext *TreeContext + leafListLock *sync.Map + stats *ImportStats +} + +func NewImportConfigProcessorParams( + intentName string, + intentPrio int32, + insertFlags *types.UpdateInsertFlags, + treeContext *TreeContext, + leafListLock *sync.Map, + stats *ImportStats, +) *ImportConfigProcessorParams { + return &ImportConfigProcessorParams{ + intentName: intentName, + intentPrio: intentPrio, + insertFlags: insertFlags, + treeContext: treeContext, + leafListLock: leafListLock, + stats: stats, + } } type ImportConfigProcessor struct { importer importer.ImportConfigAdapter insertFlags *types.UpdateInsertFlags + stats *ImportStats } func NewImportConfigProcessor(importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { return &ImportConfigProcessor{ importer: importer, insertFlags: insertFlags, + stats: NewImportStats(), } } -func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, workerPool pool.VirtualPoolI) error { +func (p *ImportConfigProcessor) GetStats() *ImportStats { + return p.stats +} + +func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { + + workerPool := poolFactory.NewVirtualPool(pool.VirtualFailFast) t := importConfigTask{ entry: e, importerElement: p.importer, - intentName: p.importer.GetName(), - intentPrio: p.importer.GetPriority(), - insertFlags: p.insertFlags, - treeContext: e.getTreeContext(), - leafListLock: &sync.Map{}, + params: NewImportConfigProcessorParams(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.getTreeContext(), &sync.Map{}, p.stats), } if err := workerPool.Submit(t); err != nil { @@ -64,16 +91,6 @@ func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, workerPool poo return nil } -func (s *sharedEntryAttributes) ImportConfig( - ctx context.Context, - importer importer.ImportConfigAdapter, - insertFlags *types.UpdateInsertFlags, - workerPool pool.VirtualPoolI, -) error { - processor := NewImportConfigProcessor(importer, insertFlags) - return processor.Run(ctx, s, workerPool) -} - func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) error) error { elem := task.entry.PathName() @@ -99,7 +116,7 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err return err } if keyChild, exists = actual.GetChild(kv); !exists { - keyChild, err = newEntry(ctx, actual, kv, task.treeContext) + keyChild, err = newEntry(ctx, actual, kv, task.params.treeContext) if err != nil { return err } @@ -108,7 +125,7 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err } // submit resolved entry with same adapter element // return importHandler(ctx, importTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext}, submit) - return submit(importConfigTask{entry: actual, importerElement: task.importerElement, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}) + return submit(importConfigTask{entry: actual, importerElement: task.importerElement, params: task.params}) } // presence container or children @@ -117,8 +134,8 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err schem := task.entry.GetSchema().GetContainer() if schem != nil && schem.IsPresence { tv := &sdcpb.TypedValue{Value: &sdcpb.TypedValue_EmptyVal{EmptyVal: &emptypb.Empty{}}} - upd := types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) - task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.insertFlags, task.entry)) + upd := types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) + task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.params.insertFlags, task.entry)) } return nil } @@ -128,12 +145,12 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err child, exists := task.entry.GetChild(childElt.GetName()) if !exists { var err error - child, err = newEntry(ctx, task.entry, childElt.GetName(), task.treeContext) + child, err = newEntry(ctx, task.entry, childElt.GetName(), task.params.treeContext) if err != nil { return fmt.Errorf("error inserting %s at %s: %w", childElt.GetName(), task.entry.SdcpbPath().ToXPath(false), err) } } - if err := submit(importConfigTask{entry: child, importerElement: childElt, intentName: task.intentName, intentPrio: task.intentPrio, insertFlags: task.insertFlags, treeContext: task.treeContext, leafListLock: task.leafListLock}); err != nil { + if err := submit(importConfigTask{entry: child, importerElement: childElt, params: task.params}); err != nil { return err } } @@ -144,8 +161,8 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err if err != nil { return err } - upd := types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) - task.entry.GetLeafVariantEntries().Add(NewLeafEntry(upd, task.insertFlags, task.entry)) + upd := types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) + task.entry.GetLeafVariantEntries().AddWithStats(NewLeafEntry(upd, task.params.insertFlags, task.entry), task.params.stats) return nil case *sdcpb.SchemaElem_Leaflist: @@ -161,7 +178,7 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err llMutex.Lock() // try storing it or load it from leafListLock - llm, loaded := task.leafListLock.LoadOrStore(task.entry.SdcpbPath().ToXPath(false), llMutex) + llm, loaded := task.params.leafListLock.LoadOrStore(task.entry.SdcpbPath().ToXPath(false), llMutex) // if it was loaded, we need to lock the loaded mutex if loaded { @@ -174,10 +191,10 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err mustAdd := false var le *LeafEntry if loaded { - le = task.entry.GetLeafVariantEntries().GetByOwner(task.intentName) + le = task.entry.GetLeafVariantEntries().GetByOwner(task.params.intentName) scalarArr = le.Value().GetLeaflistVal() } else { - le = NewLeafEntry(nil, task.insertFlags, task.entry) + le = NewLeafEntry(nil, task.params.insertFlags, task.entry) mustAdd = true scalarArr = &sdcpb.ScalarArray{Element: []*sdcpb.TypedValue{}} } @@ -190,7 +207,7 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err scalarArr.Element = append(scalarArr.Element, tv) tv = &sdcpb.TypedValue{Value: &sdcpb.TypedValue_LeaflistVal{LeaflistVal: scalarArr}} } - le.Update = types.NewUpdate(task.entry, tv, task.intentPrio, task.intentName, 0) + le.Update = types.NewUpdate(task.entry, tv, task.params.intentPrio, task.params.intentName, 0) if mustAdd { task.entry.GetLeafVariantEntries().Add(le) } @@ -200,40 +217,46 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err } } -type ImportStat struct { +type ImportStats struct { newEntries atomic.Int64 updatedEntries atomic.Int64 } -func NewImportStat() *ImportStat { - return &ImportStat{} +func NewImportStats() *ImportStats { + return &ImportStats{} } -func (is *ImportStat) String() string { +func (is *ImportStats) String() string { return fmt.Sprintf("NewEntries: %d, UpdatedEntries: %d", is.newEntries.Load(), is.updatedEntries.Load()) } -func (is *ImportStat) Join(i *ImportStat) { +func (is *ImportStats) Join(i *ImportStats) { is.newEntries.Add(i.newEntries.Load()) is.updatedEntries.Add(i.updatedEntries.Load()) } -func (is *ImportStat) AddNewEntryStat() { +func (is *ImportStats) IncrementNew() { + if is == nil { + return + } is.newEntries.Add(1) } -func (is *ImportStat) AddUpdatedEntryStat() { +func (is *ImportStats) IncrementUpdated() { + if is == nil { + return + } is.updatedEntries.Add(1) } -func (is *ImportStat) GetNewEntries() int64 { +func (is *ImportStats) GetNewCount() int64 { return is.newEntries.Load() } -func (is *ImportStat) GetUpdatedEntries() int64 { +func (is *ImportStats) GetUpdatedCount() int64 { return is.updatedEntries.Load() } -func (is *ImportStat) Changed() bool { +func (is *ImportStats) Changed() bool { return is.newEntries.Load() > 0 || is.updatedEntries.Load() > 0 } diff --git a/pkg/tree/processor_mark_owner_delete.go b/pkg/tree/processor_mark_owner_delete.go index 47b77f5c..9de8a279 100644 --- a/pkg/tree/processor_mark_owner_delete.go +++ b/pkg/tree/processor_mark_owner_delete.go @@ -23,7 +23,8 @@ func NewOwnerDeleteMarker(c *OwnerDeleteMarkerTaskConfig) *MarkOwnerDeleteProces // Run processes the entry tree starting from e, marking leaf variant entries for deletion // by the specified owner. The pool parameter should be VirtualFailFast to stop on first error. // Returns the first error encountered, or nil if successful. -func (p *MarkOwnerDeleteProcessor) Run(e Entry, pool pool.VirtualPoolI) error { +func (p *MarkOwnerDeleteProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) if err := pool.Submit(newOwnerDeleteMarkerTask(p.config, e, p.matches)); err != nil { // Clean up pool even on early error pool.CloseAndWait() diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go index 26932160..3beae7b9 100644 --- a/pkg/tree/processor_remove_deleted.go +++ b/pkg/tree/processor_remove_deleted.go @@ -48,7 +48,11 @@ func (r *RemoveDeletedProcessorParameters) GetZeroLengthLeafVariantEntries() []E // for deletion by the specified owner. The pool parameter should be VirtualFailFast // to stop on first error. // Returns the first error encountered, or nil if successful. -func (p *RemoveDeletedProcessor) Run(e Entry, pool pool.VirtualPoolI) error { +func (p *RemoveDeletedProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { + + // create a virtual task pool for removeDeleted operations + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + if err := pool.Submit(newRemoveDeletedTask(p.config, e, false)); err != nil { // Clean up pool even on early error pool.CloseAndWait() diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go index c12c1bd8..921d7b20 100644 --- a/pkg/tree/processor_reset_flags.go +++ b/pkg/tree/processor_reset_flags.go @@ -43,11 +43,14 @@ func (r *ResetFlagsProcessorParameters) GetAdjustedFlagsCount() int64 { // according to the processor configuration. The pool parameter can be either VirtualFailFast // (stops on first error) or VirtualTolerant (collects all errors). // Returns the first error for fail-fast pools, or a combined error for tolerant pools. -func (p *ResetFlagsProcessor) Run(e Entry, pool pool.VirtualPoolI) error { +func (p *ResetFlagsProcessor) Run(e Entry, poolFactory pool.VirtualPoolFactory) error { if e == nil { return fmt.Errorf("entry cannot be nil") } + // create a virtual task pool for resetFlags operations + pool := poolFactory.NewVirtualPool(pool.VirtualFailFast) + // Submit root task; workers will recursively process children if err := pool.Submit(newResetFlagsTask(p.config, e)); err != nil { // Clean up pool even on early error diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go index 929c13f2..57488969 100644 --- a/pkg/tree/processor_reset_flags_test.go +++ b/pkg/tree/processor_reset_flags_test.go @@ -101,13 +101,12 @@ func TestResetFlagsProcessorRun(t *testing.T) { // Create a virtual pool for testing taskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - virtualPool := taskPool.NewVirtualPool(pool.VirtualFailFast) root := tt.tree() fmt.Println(root.String()) - processorErr := processor.Run(root.GetRoot(), virtualPool) + processorErr := processor.Run(root.GetRoot(), taskPool) if (processorErr != nil) != tt.wantErr { t.Errorf("ResetFlagsProcessor.Run() error = %v, wantErr %v", processorErr, tt.wantErr) return diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index de2c80e0..5efeb36d 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -88,7 +88,7 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error { +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error { r.treeContext.SetActualOwner(importer.GetName()) e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) @@ -101,7 +101,7 @@ func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, impo // store explicit deletes r.explicitDeletes.Add(importer.GetName(), importer.GetPriority(), importer.GetDeletes()) - return e.ImportConfig(ctx, importer, flags, pool) + return e.ImportConfig(ctx, importer, flags, poolFactory) } func (r *RootEntry) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 7a69ffef..4f7d832f 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -522,8 +522,8 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags(), vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatal(err) } @@ -634,7 +634,6 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { t.Logf("Want:\n%s", wantStr) t.Logf("Got:\n%s", resultRoot.String()) } - }) } } diff --git a/pkg/tree/sharedEntryAttributes.go b/pkg/tree/sharedEntryAttributes.go index 60bc58a1..0974369a 100644 --- a/pkg/tree/sharedEntryAttributes.go +++ b/pkg/tree/sharedEntryAttributes.go @@ -14,6 +14,8 @@ import ( "unicode/utf8" "github.com/sdcio/data-server/pkg/config" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" logf "github.com/sdcio/logger" @@ -121,7 +123,7 @@ func (s *sharedEntryAttributes) GetRoot() Entry { return s.parent.GetRoot() } -func (s sharedEntryAttributes) getTreeContext() *TreeContext { +func (s *sharedEntryAttributes) getTreeContext() *TreeContext { return s.treeContext } @@ -1742,3 +1744,8 @@ func (s *sharedEntryAttributes) containsOnlyDefaults() bool { func (s *sharedEntryAttributes) GetLeafVariantEntries() LeafVariantEntries { return s.leafVariants } + +func (s *sharedEntryAttributes) ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error { + processor := NewImportConfigProcessor(importer, insertFlags) + return processor.Run(ctx, s, poolFactory) +} diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index fa7501fd..ba27dacd 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -146,8 +146,8 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Error(err) } @@ -655,8 +655,8 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -778,8 +778,8 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -1133,8 +1133,8 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vpf) if err != nil { t.Error(err) return @@ -1142,10 +1142,9 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { // mark owner delete sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return @@ -1286,8 +1285,8 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -1456,8 +1455,8 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 3a32d32f..e56b957a 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -238,8 +238,8 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 2ce12413..0ffabd7b 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -68,8 +68,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) } @@ -183,10 +182,9 @@ func TestValidate_RangesSigned(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) @@ -320,10 +318,8 @@ func TestValidate_RangesUnSigned(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - importPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) - // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), importPool) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index 398ff87e..6ffe57d3 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -594,10 +594,9 @@ func TestToXMLTable(t *testing.T) { if tt.newConfig != nil { sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - deleteVisitorPool := sharedTaskPool.NewVirtualPool(pool.VirtualFailFast) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner, false)) - err = ownerDeleteMarker.Run(root.GetRoot(), deleteVisitorPool) + err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { t.Error(err) return diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 498a90c8..acfb643d 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -140,7 +140,7 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem } type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error } func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { @@ -158,8 +158,8 @@ func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root Root return err } - vp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()).NewVirtualPool(pool.VirtualFailFast) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vp) + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vpf) if err != nil { return err } From e243d708424c773e96bc8b3f78385e8699cd0e67 Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 2 Feb 2026 16:20:14 +0100 Subject: [PATCH 11/17] update --- mocks/mocktreeentry/entry.go | 156 ++++++------ pkg/datastore/intent_rpc.go | 2 +- pkg/datastore/sync.go | 2 +- pkg/datastore/sync_test.go | 8 +- pkg/datastore/target/gnmi/stream.go | 2 +- pkg/datastore/transaction_rpc.go | 20 +- pkg/datastore/transaction_rpc_test.go | 226 ++++++++++++++++++ pkg/datastore/tree_operation_test.go | 2 +- .../tree_operation_validation_test.go | 2 +- pkg/datastore/types/transaction_intent.go | 9 + pkg/tree/entry.go | 29 +-- pkg/tree/entry_map.go | 4 +- .../proto/proto_tree_importer_test.go | 4 +- .../importer/xml/xml_tree_importer_test.go | 2 +- pkg/tree/json.go | 5 +- pkg/tree/leaf_entry.go | 2 +- pkg/tree/leaf_variants.go | 2 +- pkg/tree/processor_blame_config.go | 3 +- pkg/tree/processor_blame_config_test.go | 10 +- pkg/tree/processor_importer.go | 84 +++---- pkg/tree/processor_mark_owner_delete.go | 3 +- pkg/tree/processor_remove_deleted.go | 5 +- pkg/tree/processor_reset_flags.go | 3 +- pkg/tree/processor_validate.go | 4 +- pkg/tree/root_entry.go | 36 +-- pkg/tree/root_entry_test.go | 12 +- pkg/tree/sharedEntryAttributes.go | 97 ++++---- pkg/tree/sharedEntryAttributes_test.go | 34 +-- pkg/tree/sorter.go | 6 +- pkg/tree/tree_context.go | 12 + pkg/tree/types/descend_methods.go | 8 + pkg/tree/types/import_stats.go | 50 ++++ pkg/tree/validation_entry_leafref.go | 2 +- pkg/tree/validation_entry_leafref_test.go | 2 +- pkg/tree/validation_range_test.go | 6 +- pkg/tree/visitor_base.go | 6 +- pkg/tree/visitor_explicit_delete_test.go | 12 +- pkg/tree/xml.go | 3 +- pkg/utils/testhelper/utils.go | 14 +- 39 files changed, 590 insertions(+), 299 deletions(-) create mode 100644 pkg/datastore/transaction_rpc_test.go create mode 100644 pkg/tree/types/descend_methods.go create mode 100644 pkg/tree/types/import_stats.go diff --git a/mocks/mocktreeentry/entry.go b/mocks/mocktreeentry/entry.go index 1ee421e0..543cd674 100644 --- a/mocks/mocktreeentry/entry.go +++ b/mocks/mocktreeentry/entry.go @@ -15,9 +15,7 @@ import ( etree "github.com/beevik/etree" config "github.com/sdcio/data-server/pkg/config" - pool "github.com/sdcio/data-server/pkg/pool" tree "github.com/sdcio/data-server/pkg/tree" - importer "github.com/sdcio/data-server/pkg/tree/importer" types "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" tree_persist "github.com/sdcio/sdc-protos/tree_persist" @@ -48,6 +46,20 @@ func (m *MockEntry) EXPECT() *MockEntryMockRecorder { return m.recorder } +// AddChild mocks base method. +func (m *MockEntry) AddChild(arg0 context.Context, arg1 tree.Entry) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChild", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChild indicates an expected call of AddChild. +func (mr *MockEntryMockRecorder) AddChild(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChild", reflect.TypeOf((*MockEntry)(nil).AddChild), arg0, arg1) +} + // AddUpdateRecursive mocks base method. func (m *MockEntry) AddUpdateRecursive(ctx context.Context, relativePath *sdcpb.Path, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() @@ -78,6 +90,20 @@ func (mr *MockEntryMockRecorder) BreadthSearch(ctx, path any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BreadthSearch", reflect.TypeOf((*MockEntry)(nil).BreadthSearch), ctx, path) } +// CanDeleteBranch mocks base method. +func (m *MockEntry) CanDeleteBranch(keepDefault bool) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanDeleteBranch", keepDefault) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanDeleteBranch indicates an expected call of CanDeleteBranch. +func (mr *MockEntryMockRecorder) CanDeleteBranch(keepDefault any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanDeleteBranch", reflect.TypeOf((*MockEntry)(nil).CanDeleteBranch), keepDefault) +} + // DeepCopy mocks base method. func (m *MockEntry) DeepCopy(tc *tree.TreeContext, parent tree.Entry) (tree.Entry, error) { m.ctrl.T.Helper() @@ -107,6 +133,18 @@ func (mr *MockEntryMockRecorder) DeleteBranch(ctx, path, owner any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBranch", reflect.TypeOf((*MockEntry)(nil).DeleteBranch), ctx, path, owner) } +// DeleteCanDeleteChilds mocks base method. +func (m *MockEntry) DeleteCanDeleteChilds(keepDefault bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "DeleteCanDeleteChilds", keepDefault) +} + +// DeleteCanDeleteChilds indicates an expected call of DeleteCanDeleteChilds. +func (mr *MockEntryMockRecorder) DeleteCanDeleteChilds(keepDefault any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCanDeleteChilds", reflect.TypeOf((*MockEntry)(nil).DeleteCanDeleteChilds), keepDefault) +} + // FilterChilds mocks base method. func (m *MockEntry) FilterChilds(keys map[string]string) ([]tree.Entry, error) { m.ctrl.T.Helper() @@ -166,7 +204,7 @@ func (mr *MockEntryMockRecorder) GetChild(name any) *gomock.Call { } // GetChilds mocks base method. -func (m *MockEntry) GetChilds(arg0 tree.DescendMethod) tree.EntryMap { +func (m *MockEntry) GetChilds(arg0 types.DescendMethod) tree.EntryMap { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetChilds", arg0) ret0, _ := ret[0].(tree.EntryMap) @@ -348,32 +386,32 @@ func (mr *MockEntryMockRecorder) GetSchemaKeys() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaKeys", reflect.TypeOf((*MockEntry)(nil).GetSchemaKeys)) } -// HoldsLeafvariants mocks base method. -func (m *MockEntry) HoldsLeafvariants() bool { +// GetTreeContext mocks base method. +func (m *MockEntry) GetTreeContext() *tree.TreeContext { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HoldsLeafvariants") - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "GetTreeContext") + ret0, _ := ret[0].(*tree.TreeContext) return ret0 } -// HoldsLeafvariants indicates an expected call of HoldsLeafvariants. -func (mr *MockEntryMockRecorder) HoldsLeafvariants() *gomock.Call { +// GetTreeContext indicates an expected call of GetTreeContext. +func (mr *MockEntryMockRecorder) GetTreeContext() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HoldsLeafvariants", reflect.TypeOf((*MockEntry)(nil).HoldsLeafvariants)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTreeContext", reflect.TypeOf((*MockEntry)(nil).GetTreeContext)) } -// ImportConfig mocks base method. -func (m *MockEntry) ImportConfig(ctx context.Context, arg1 importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, pool pool.VirtualPoolI) error { +// HoldsLeafvariants mocks base method. +func (m *MockEntry) HoldsLeafvariants() bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ImportConfig", ctx, arg1, flags, pool) - ret0, _ := ret[0].(error) + ret := m.ctrl.Call(m, "HoldsLeafvariants") + ret0, _ := ret[0].(bool) return ret0 } -// ImportConfig indicates an expected call of ImportConfig. -func (mr *MockEntryMockRecorder) ImportConfig(ctx, arg1, flags, pool any) *gomock.Call { +// HoldsLeafvariants indicates an expected call of HoldsLeafvariants. +func (mr *MockEntryMockRecorder) HoldsLeafvariants() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportConfig", reflect.TypeOf((*MockEntry)(nil).ImportConfig), ctx, arg1, flags, pool) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HoldsLeafvariants", reflect.TypeOf((*MockEntry)(nil).HoldsLeafvariants)) } // IsRoot mocks base method. @@ -434,6 +472,20 @@ func (mr *MockEntryMockRecorder) PathName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PathName", reflect.TypeOf((*MockEntry)(nil).PathName)) } +// RemainsToExist mocks base method. +func (m *MockEntry) RemainsToExist() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemainsToExist") + ret0, _ := ret[0].(bool) + return ret0 +} + +// RemainsToExist indicates an expected call of RemainsToExist. +func (mr *MockEntryMockRecorder) RemainsToExist() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemainsToExist", reflect.TypeOf((*MockEntry)(nil).RemainsToExist)) +} + // SdcpbPath mocks base method. func (m *MockEntry) SdcpbPath() *sdcpb.Path { m.ctrl.T.Helper() @@ -548,20 +600,6 @@ func (mr *MockEntryMockRecorder) Walk(ctx, v any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockEntry)(nil).Walk), ctx, v) } -// addChild mocks base method. -func (m *MockEntry) addChild(arg0 context.Context, arg1 tree.Entry) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "addChild", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// addChild indicates an expected call of addChild. -func (mr *MockEntryMockRecorder) addChild(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "addChild", reflect.TypeOf((*MockEntry)(nil).addChild), arg0, arg1) -} - // addUpdateRecursiveInternal mocks base method. func (m *MockEntry) addUpdateRecursiveInternal(ctx context.Context, path *sdcpb.Path, idx int, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() @@ -591,32 +629,6 @@ func (mr *MockEntryMockRecorder) canDelete() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "canDelete", reflect.TypeOf((*MockEntry)(nil).canDelete)) } -// canDeleteBranch mocks base method. -func (m *MockEntry) canDeleteBranch(keepDefault bool) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "canDeleteBranch", keepDefault) - ret0, _ := ret[0].(bool) - return ret0 -} - -// canDeleteBranch indicates an expected call of canDeleteBranch. -func (mr *MockEntryMockRecorder) canDeleteBranch(keepDefault any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "canDeleteBranch", reflect.TypeOf((*MockEntry)(nil).canDeleteBranch), keepDefault) -} - -// deleteCanDeleteChilds mocks base method. -func (m *MockEntry) deleteCanDeleteChilds(keepDefault bool) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "deleteCanDeleteChilds", keepDefault) -} - -// deleteCanDeleteChilds indicates an expected call of deleteCanDeleteChilds. -func (mr *MockEntryMockRecorder) deleteCanDeleteChilds(keepDefault any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "deleteCanDeleteChilds", reflect.TypeOf((*MockEntry)(nil).deleteCanDeleteChilds), keepDefault) -} - // getHighestPrecedenceLeafValue mocks base method. func (m *MockEntry) getHighestPrecedenceLeafValue(arg0 context.Context) (*tree.LeafEntry, error) { m.ctrl.T.Helper() @@ -661,20 +673,6 @@ func (mr *MockEntryMockRecorder) getOrCreateChilds(ctx, path any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getOrCreateChilds", reflect.TypeOf((*MockEntry)(nil).getOrCreateChilds), ctx, path) } -// remainsToExist mocks base method. -func (m *MockEntry) remainsToExist() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "remainsToExist") - ret0, _ := ret[0].(bool) - return ret0 -} - -// remainsToExist indicates an expected call of remainsToExist. -func (mr *MockEntryMockRecorder) remainsToExist() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remainsToExist", reflect.TypeOf((*MockEntry)(nil).remainsToExist)) -} - // shouldDelete mocks base method. func (m *MockEntry) shouldDelete() bool { m.ctrl.T.Helper() @@ -768,10 +766,10 @@ func (m *MockEntryVisitor) EXPECT() *MockEntryVisitorMockRecorder { } // DescendMethod mocks base method. -func (m *MockEntryVisitor) DescendMethod() tree.DescendMethod { +func (m *MockEntryVisitor) DescendMethod() types.DescendMethod { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DescendMethod") - ret0, _ := ret[0].(tree.DescendMethod) + ret0, _ := ret[0].(types.DescendMethod) return ret0 } @@ -923,6 +921,18 @@ func (mr *MockLeafVariantEntriesMockRecorder) AddExplicitDeleteEntry(owner, prio return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddExplicitDeleteEntry", reflect.TypeOf((*MockLeafVariantEntries)(nil).AddExplicitDeleteEntry), owner, priority) } +// AddWithStats mocks base method. +func (m *MockLeafVariantEntries) AddWithStats(l *tree.LeafEntry, stats *types.ImportStats) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddWithStats", l, stats) +} + +// AddWithStats indicates an expected call of AddWithStats. +func (mr *MockLeafVariantEntriesMockRecorder) AddWithStats(l, stats any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWithStats", reflect.TypeOf((*MockLeafVariantEntries)(nil).AddWithStats), l, stats) +} + // DeleteByOwner mocks base method. func (m *MockLeafVariantEntries) DeleteByOwner(owner string) *tree.LeafEntry { m.ctrl.T.Helper() diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 4d740f3a..dd9db188 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -77,7 +77,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } protoImporter := proto.NewProtoTreeImporter(tp) - err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool) + _, err = root.ImportConfig(ctx, nil, protoImporter, types.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 8a411d4f..17d1a0ec 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -39,7 +39,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i // import new config if provided if importer != nil { - err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), d.taskPool) + _, err := d.syncTree.ImportConfig(ctx, &sdcpb.Path{}, importer, treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return err } diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index abd5c84c..1129d047 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -81,7 +81,7 @@ func TestApplyToRunning(t *testing.T) { json.Unmarshal([]byte(confStr), &v) vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -187,7 +187,7 @@ func TestApplyToRunning(t *testing.T) { json.Unmarshal([]byte(confStr), &v) vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -291,7 +291,7 @@ func TestApplyToRunning(t *testing.T) { json.Unmarshal([]byte(confStr), &v) vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } @@ -400,7 +400,7 @@ func TestApplyToRunning(t *testing.T) { d := tt.resultFunc() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) + _, err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) } diff --git a/pkg/datastore/target/gnmi/stream.go b/pkg/datastore/target/gnmi/stream.go index 96d42067..546e2119 100644 --- a/pkg/datastore/target/gnmi/stream.go +++ b/pkg/datastore/target/gnmi/stream.go @@ -148,7 +148,7 @@ func (s *StreamSync) buildTreeSyncWithDatastore(cUS <-chan *NotificationData, sy if err != nil { log.Error(err, "failed adding update to synctree") } - syncTree.AddExplicitDeletes(tree.RunningIntentName, tree.RunningValuesPrio, noti.deletes) + syncTree.GetTreeContext().AddExplicitDeletes(tree.RunningIntentName, tree.RunningValuesPrio, noti.deletes) case <-syncResponse: syncTree, err = s.syncToRunning(syncTree, syncTreeMutex, true) tickerActive = true diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 21a8fe76..0e76b97b 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -46,6 +46,9 @@ func (d *Datastore) SdcpbTransactionIntentToInternalTI(ctx context.Context, req if req.GetDeleteIgnoreNoExist() { ti.SetDeleteIgnoreNonExisting() } + if req.GetPreviouslyApplied() { + ti.SetPreviouslyApplied() + } // convert the sdcpb.updates to tree.UpdateSlice Updates, err := treetypes.ExpandAndConvertIntent(ctx, d.schemaClient, req.GetIntent(), req.GetPriority(), req.GetUpdate(), time.Now().Unix()) @@ -85,7 +88,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa if err != nil { return nil, err } - err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool) + _, err = root.ImportConfig(ctx, nil, treeproto.NewProtoTreeImporter(runningProto), treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -178,7 +181,7 @@ func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.Roo intentNames = append(intentNames, intent.GetIntentName()) protoLoader := treeproto.NewProtoTreeImporter(intent) - err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) + _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) if err != nil { return nil, err } @@ -209,6 +212,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ flagNew := treetypes.NewUpdateInsertFlags() // where the New flag is set flagNew.SetNewFlag() + flagExisting := treetypes.NewUpdateInsertFlags() // iterate through all the intents for _, intent := range transaction.GetNewIntents() { @@ -226,7 +230,7 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ } // clear the owners existing explicit delete entries, retrieving the old entries for storing in the transaction for possible rollback - oldExplicitDeletes := root.RemoveExplicitDeletes(intent.GetName()) + oldExplicitDeletes := root.GetTreeContext().RemoveExplicitDeletes(intent.GetName()) priority := int32(math.MaxInt32) if len(oldIntentContent) > 0 { @@ -240,14 +244,20 @@ func (d *Datastore) lowlevelTransactionSet(ctx context.Context, transaction *typ } if !intent.GetDeleteFlag() { + flag := flagNew + // determine the correct flag to use based on whether the intent is non-revertive and was previously applied + if intent.NonRevertive() && intent.GetPreviouslyApplied() { + flag = flagExisting + } + // add the content to the Tree - err = root.AddUpdatesRecursive(ctx, intent.GetUpdates(), flagNew) + err = root.AddUpdatesRecursive(ctx, intent.GetUpdates(), flag) if err != nil { return nil, err } // add the explicit delete entries - root.AddExplicitDeletes(intent.GetName(), intent.GetPriority(), intent.GetDeletes()) + root.GetTreeContext().AddExplicitDeletes(intent.GetName(), intent.GetPriority(), intent.GetDeletes()) } root.SetNonRevertiveIntent(intent.GetName(), intent.NonRevertive()) diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go new file mode 100644 index 00000000..c25adadd --- /dev/null +++ b/pkg/datastore/transaction_rpc_test.go @@ -0,0 +1,226 @@ +package datastore + +import ( + "context" + "encoding/json" + "runtime" + "sync" + "testing" + "time" + + "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/mocks/mockcacheclient" + "github.com/sdcio/data-server/mocks/mocktarget" + "github.com/sdcio/data-server/pkg/config" + schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/datastore/types" + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + treetypes "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils/testhelper" + sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" + "go.uber.org/mock/gomock" +) + +func TestTransactionSet_PreviouslyApplied(t *testing.T) { + ctx := context.Background() + + // Setup Schema + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatal(err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + + // Setup Running Config Data + runningDevice := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("my description"), + }, + }, + } + runningJson, err := ygot.EmitJSON(runningDevice, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + if err != nil { + t.Fatalf("failed to marshal running config: %v", err) + } + var runningAny any + json.Unmarshal([]byte(runningJson), &runningAny) + + // Setup Intent Data (Same as Running) + intentStrSame := runningJson // Same content + + // Setup Intent Data (Different) + deviceDiff := &sdcio_schema.Device{ + Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + "ethernet-1/1": { + Name: ygot.String("ethernet-1/1"), + Description: ygot.String("new description"), + }, + }, + } + intentStrDiff, _ := ygot.EmitJSON(deviceDiff, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: false, + }) + + tests := []struct { + name string + previouslyApplied bool + nonRevertive bool + intentStr string + expectUpdates bool + }{ + { + name: "Revertive - Not Previously Applied - Should Produce Updates (Redundant)", + previouslyApplied: false, + nonRevertive: false, + intentStr: intentStrSame, + expectUpdates: true, + }, + { + name: "Revertive - Previously Applied - Should Produce Updates (PA Ignored)", + previouslyApplied: true, + nonRevertive: false, + intentStr: intentStrSame, + expectUpdates: true, + }, + { + name: "Revertive - Previously Applied - But Value Changed - Should Produce Updates", + previouslyApplied: true, + nonRevertive: false, + intentStr: intentStrDiff, + expectUpdates: true, + }, + { + name: "NonRevertive - Previously Applied - Value Changed - Should Produce NO Updates", + previouslyApplied: true, + nonRevertive: true, + intentStr: intentStrDiff, + expectUpdates: false, + }, + { + name: "NonRevertive - Not Previously Applied - Value Changed - Should Produce Updates", + previouslyApplied: false, + nonRevertive: true, + intentStr: intentStrDiff, + expectUpdates: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Setup Mock Cache Client + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + // Expect IntentGetAll (called by LoadAllButRunningIntents) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + close(intentChan) + close(errChan) + }).AnyTimes() + + // Expect IntentModify (called by TransactionSet to save intent) + ccb.EXPECT().IntentModify(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + // Setup Mock SBI + sbi := mocktarget.NewMockTarget(ctrl) + // Expect Set if updates are expected or if dryRun is false (we will use dryRun=false) + // Actually TransactionSet calls applyIntent which calls sbi.Set + sbi.EXPECT().Set(gomock.Any(), gomock.Any()).Return(&sdcpb.SetDataResponse{}, nil).AnyTimes() + + // Setup SyncTree with Running Config + tc := tree.NewTreeContext(scb, tree.RunningIntentName) + syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) + if err != nil { + t.Fatalf("failed to create sync tree root: %v", err) + } + vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + // Populate SyncTree with Running config + _, err = syncTreeRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(runningAny, tree.RunningIntentName, tree.RunningValuesPrio, false), treetypes.NewUpdateInsertFlags(), vpf) + if err != nil { + t.Fatalf("failed to import running config: %v", err) + } + err = syncTreeRoot.FinishInsertionPhase(ctx) + if err != nil { + t.Fatalf("failed to finish insertion phase: %v", err) + } + // Reset flags on syncTree so everything is "existing" + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) + err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTreeRoot.GetRoot(), vpf) + if err != nil { + t.Fatalf("failed to reset flags: %v", err) + } + + // Setup Datastore + ds := &Datastore{ + config: &config.DatastoreConfig{ + Validation: config.NewValidationConfig(), + Name: "test-ds", + }, + syncTreeMutex: &sync.RWMutex{}, + syncTree: syncTreeRoot, // Pre-populated syncTree + taskPool: vpf, + cacheClient: ccb, + sbi: sbi, + dmutex: &sync.Mutex{}, + schemaClient: scb, + } + ds.transactionManager = types.NewTransactionManager(NewDatastoreRollbackAdapter(ds)) + + // Prepare Transaction Request + transactionId := "txn-1" + + // Build TransactionIntent from req using helper or manually + // We need types.TransactionIntent + intentName := "intent1" + priority := int32(10) + + ti := types.NewTransactionIntent(intentName, priority) + if tt.previouslyApplied { + ti.SetPreviouslyApplied() + } + if tt.nonRevertive { + ti.SetNonRevertive() + } + + // Parse updates from intentStr + updates, err := treetypes.ExpandAndConvertIntent(ctx, scb, intentName, priority, []*sdcpb.Update{{ + Path: &sdcpb.Path{}, + Value: &sdcpb.TypedValue{ + Value: &sdcpb.TypedValue_JsonVal{ + JsonVal: []byte(tt.intentStr), + }, + }, + }}, time.Now().Unix()) + if err != nil { + t.Fatalf("failed to expand intent: %v", err) + } + ti.AddUpdates(updates) + + transactionIntents := []*types.TransactionIntent{ti} + + // Call TransactionSet + resp, err := ds.TransactionSet(ctx, transactionId, transactionIntents, nil, 10*time.Second, false) + if err != nil { + t.Fatalf("TransactionSet failed: %v", err) + } + + // Verify Updates + hasUpdates := len(resp.GetUpdate()) > 0 + if hasUpdates != tt.expectUpdates { + t.Errorf("Expected updates: %v, got: %v (count: %d)\nUpdates: %v", tt.expectUpdates, hasUpdates, len(resp.GetUpdate()), resp.GetUpdate()) + } + }) + } +} diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 9fd1efa5..0ecd35bd 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1519,7 +1519,7 @@ func TestDatastore_populateTree(t *testing.T) { sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, sharedPool) + _, err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, sharedPool) if err != nil { t.Error(err) diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index 61a2ada4..14400854 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -199,7 +199,7 @@ func TestDatastore_validateTree(t *testing.T) { importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, path, importer, flagsNew, vpf) + _, err = root.ImportConfig(ctx, path, importer, flagsNew, vpf) if err != nil { t.Error(err) } diff --git a/pkg/datastore/types/transaction_intent.go b/pkg/datastore/types/transaction_intent.go index db65f307..674a7ec1 100644 --- a/pkg/datastore/types/transaction_intent.go +++ b/pkg/datastore/types/transaction_intent.go @@ -16,6 +16,7 @@ type TransactionIntent struct { nonRevertive bool deleteIgnoreNonExisting bool explicitDeletes *sdcpb.PathSet + previouslyApplied bool } func NewTransactionIntent(name string, priority int32) *TransactionIntent { @@ -51,6 +52,10 @@ func (ti *TransactionIntent) GetDeletes() *sdcpb.PathSet { return ti.explicitDeletes } +func (ti *TransactionIntent) GetPreviouslyApplied() bool { + return ti.previouslyApplied +} + func (ti *TransactionIntent) GetOnlyIntended() bool { return ti.onlyIntended } @@ -75,6 +80,10 @@ func (ti *TransactionIntent) SetDeleteFlag() { ti.delete = true } +func (ti *TransactionIntent) SetPreviouslyApplied() { + ti.previouslyApplied = true +} + func (ti *TransactionIntent) SetDeleteOnlyIntendedFlag() { ti.delete = true ti.onlyIntended = true diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 57eff845..6efe1ad8 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -6,12 +6,9 @@ import ( "github.com/beevik/etree" "github.com/sdcio/data-server/pkg/config" - "github.com/sdcio/data-server/pkg/pool" - "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" - "github.com/sdcio/sdc-protos/tree_persist" - sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "github.com/sdcio/sdc-protos/tree_persist" ) const ( @@ -24,8 +21,8 @@ const ( ReplaceIntentName = "replace" ) -// newEntry constructor for Entries -func newEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeContext) (*sharedEntryAttributes, error) { +// NewEntry constructor for Entries +func NewEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeContext) (*sharedEntryAttributes, error) { // create a new sharedEntryAttributes instance sea, err := newSharedEntryAttributes(ctx, parent, pathElemName, tc) if err != nil { @@ -33,7 +30,7 @@ func newEntry(ctx context.Context, parent Entry, pathElemName string, tc *TreeCo } // add the Entry as a child to the parent Entry - err = parent.addChild(ctx, sea) + err = parent.AddChild(ctx, sea) return sea, err } @@ -44,7 +41,7 @@ type Entry interface { // GetLevel returns the depth of the Entry in the tree GetLevel() int // addChild Add a child entry - addChild(context.Context, Entry) error + AddChild(context.Context, Entry) error // getOrCreateChilds retrieves the sub-child pointed at by the path. // if the path does not exist in its full extend, the entries will be added along the way // if the path does not point to a schema defined path an error will be raise @@ -107,7 +104,7 @@ type Entry interface { // as part of the SetIntent process. We need to consider this, when evaluating e.g. LeafRefs. // The returned boolean will in indicate if the value remains existing (true) after the setintent. // Or will disappear from device (running) as part of the update action. - remainsToExist() bool + RemainsToExist() bool // shouldDelete returns true if an explicit delete should be issued for the given branch shouldDelete() bool // canDelete checks if the entry can be Deleted. @@ -116,7 +113,7 @@ type Entry interface { // - remainsToExists() returns true, because they remain to exist even though implicitly. // - shouldDelete() returns false, because no explicit delete should be issued for them. canDelete() bool - GetChilds(DescendMethod) EntryMap + GetChilds(types.DescendMethod) EntryMap GetChild(name string) (Entry, bool) // entry, exists FilterChilds(keys map[string]string) ([]Entry, error) // ToJson returns the Tree contained structure as JSON @@ -132,7 +129,7 @@ type Entry interface { ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error + // ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) @@ -146,13 +143,13 @@ type Entry interface { // returns true if the Entry contains leafvariants (presence container, field or leaflist) HoldsLeafvariants() bool - canDeleteBranch(keepDefault bool) bool - deleteCanDeleteChilds(keepDefault bool) - getTreeContext() *TreeContext + CanDeleteBranch(keepDefault bool) bool + DeleteCanDeleteChilds(keepDefault bool) + GetTreeContext() *TreeContext } type EntryVisitor interface { - DescendMethod() DescendMethod + DescendMethod() types.DescendMethod Visit(ctx context.Context, e Entry) error Up() } @@ -173,7 +170,7 @@ type LeafVariantEntries interface { GetByOwner(owner string) *LeafEntry RemoveDeletedByOwner(owner string) *LeafEntry Add(l *LeafEntry) - AddWithStats(l *LeafEntry, stats *ImportStats) + AddWithStats(l *LeafEntry, stats *types.ImportStats) Length() int } diff --git a/pkg/tree/entry_map.go b/pkg/tree/entry_map.go index cd94aa28..59fa6832 100644 --- a/pkg/tree/entry_map.go +++ b/pkg/tree/entry_map.go @@ -1,6 +1,8 @@ package tree -import "sort" +import ( + "sort" +) type EntryMap map[string]Entry diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 14750a7c..b646292a 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -131,7 +131,7 @@ func TestProtoTreeImporter(t *testing.T) { jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vpf) + _, err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatal(err) } @@ -158,7 +158,7 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) vpf2 := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vpf2) + _, err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vpf2) if err != nil { t.Error(err) } diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index dc1ad823..a2ae86e3 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -96,7 +96,7 @@ func TestXmlTreeImporter(t *testing.T) { sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), sharedPool) + _, err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), sharedPool) sharedPool.CloseForSubmit() sharedPool.Wait() diff --git a/pkg/tree/json.go b/pkg/tree/json.go index 23abe03b..151f2f24 100644 --- a/pkg/tree/json.go +++ b/pkg/tree/json.go @@ -4,6 +4,7 @@ import ( "fmt" "slices" + "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) @@ -41,7 +42,7 @@ func (s *sharedEntryAttributes) toJsonInternal(onlyNewOrUpdated bool, ietf bool) // ancestor is a list with keys. result := map[string]any{} - for key, c := range s.GetChilds(DescendMethodActiveChilds) { + for key, c := range s.GetChilds(types.DescendMethodActiveChilds) { ancest, _ := s.GetFirstAncestorWithSchema() prefixedKey := jsonGetIetfPrefixConditional(key, c, ancest, ietf) // recurse the call @@ -101,7 +102,7 @@ func (s *sharedEntryAttributes) toJsonInternal(onlyNewOrUpdated bool, ietf bool) default: // otherwise this is a map result := map[string]any{} - for key, c := range s.GetChilds(DescendMethodActiveChilds) { + for key, c := range s.GetChilds(types.DescendMethodActiveChilds) { prefixedKey := jsonGetIetfPrefixConditional(key, c, s, ietf) js, err := c.toJsonInternal(onlyNewOrUpdated, ietf) if err != nil { diff --git a/pkg/tree/leaf_entry.go b/pkg/tree/leaf_entry.go index fef0c61c..8abab2ba 100644 --- a/pkg/tree/leaf_entry.go +++ b/pkg/tree/leaf_entry.go @@ -146,7 +146,7 @@ func (l *LeafEntry) NonRevertive() bool { if l.parentEntry == nil { return false } - return l.parentEntry.getTreeContext().IsNonRevertiveIntent(l.Owner()) + return l.parentEntry.GetTreeContext().IsNonRevertiveIntent(l.Owner()) } // String returns a string representation of the LeafEntry diff --git a/pkg/tree/leaf_variants.go b/pkg/tree/leaf_variants.go index b0709aaa..ced69248 100644 --- a/pkg/tree/leaf_variants.go +++ b/pkg/tree/leaf_variants.go @@ -25,7 +25,7 @@ func newLeafVariants(tc *TreeContext, parentEnty Entry) *LeafVariants { } } -func (lv *LeafVariants) AddWithStats(le *LeafEntry, stats *ImportStats) { +func (lv *LeafVariants) AddWithStats(le *LeafEntry, stats *types.ImportStats) { if leafVariant := lv.GetByOwner(le.Owner()); leafVariant != nil { if leafVariant.Update.Equal(le.Update) { // it seems like the element was not deleted, so drop the delete flag diff --git a/pkg/tree/processor_blame_config.go b/pkg/tree/processor_blame_config.go index 9c42c918..603b4ca7 100644 --- a/pkg/tree/processor_blame_config.go +++ b/pkg/tree/processor_blame_config.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/protobuf/proto" ) @@ -94,7 +95,7 @@ func (t *BlameConfigTask) Run(ctx context.Context, submit func(pool.Task) error) } } - childs := t.selfEntry.GetChilds(DescendMethodActiveChilds) + childs := t.selfEntry.GetChilds(types.DescendMethodActiveChilds) for _, childKey := range childs.SortedKeys() { childEntry := childs[childKey] childHighestLe := childEntry.GetLeafVariantEntries().GetHighestPrecedence(false, true, true) diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index 007ce747..f34e5f24 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -46,7 +46,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -111,13 +111,13 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } conf2 := config2() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -132,7 +132,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/processor_importer.go b/pkg/tree/processor_importer.go index 281ee9d2..3c0c7a06 100644 --- a/pkg/tree/processor_importer.go +++ b/pkg/tree/processor_importer.go @@ -5,7 +5,6 @@ import ( "fmt" "slices" "sync" - "sync/atomic" "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/importer" @@ -26,7 +25,7 @@ type ImportConfigProcessorParams struct { insertFlags *types.UpdateInsertFlags treeContext *TreeContext leafListLock *sync.Map - stats *ImportStats + stats *types.ImportStats } func NewImportConfigProcessorParams( @@ -35,7 +34,7 @@ func NewImportConfigProcessorParams( insertFlags *types.UpdateInsertFlags, treeContext *TreeContext, leafListLock *sync.Map, - stats *ImportStats, + stats *types.ImportStats, ) *ImportConfigProcessorParams { return &ImportConfigProcessorParams{ intentName: intentName, @@ -50,29 +49,37 @@ func NewImportConfigProcessorParams( type ImportConfigProcessor struct { importer importer.ImportConfigAdapter insertFlags *types.UpdateInsertFlags - stats *ImportStats + stats *types.ImportStats } func NewImportConfigProcessor(importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { return &ImportConfigProcessor{ importer: importer, insertFlags: insertFlags, - stats: NewImportStats(), + stats: types.NewImportStats(), } } -func (p *ImportConfigProcessor) GetStats() *ImportStats { +func (p *ImportConfigProcessor) GetStats() *types.ImportStats { return p.stats } func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { + // set actual owner + e.GetTreeContext().SetActualOwner(p.importer.GetName()) + + // store non revertive info + e.GetTreeContext().nonRevertiveInfo[p.importer.GetName()] = p.importer.GetNonRevertive() + + // store explicit deletes + e.GetTreeContext().explicitDeletes.Add(p.importer.GetName(), p.importer.GetPriority(), p.importer.GetDeletes()) workerPool := poolFactory.NewVirtualPool(pool.VirtualFailFast) t := importConfigTask{ entry: e, importerElement: p.importer, - params: NewImportConfigProcessorParams(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.getTreeContext(), &sync.Map{}, p.stats), + params: NewImportConfigProcessorParams(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.GetTreeContext(), &sync.Map{}, p.stats), } if err := workerPool.Submit(t); err != nil { @@ -116,7 +123,7 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err return err } if keyChild, exists = actual.GetChild(kv); !exists { - keyChild, err = newEntry(ctx, actual, kv, task.params.treeContext) + keyChild, err = NewEntry(ctx, actual, kv, task.params.treeContext) if err != nil { return err } @@ -140,18 +147,27 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err return nil } - // submit each child (no external locking per your guarantee) + // submit each child for _, childElt := range elems { child, exists := task.entry.GetChild(childElt.GetName()) if !exists { var err error - child, err = newEntry(ctx, task.entry, childElt.GetName(), task.params.treeContext) + child, err = NewEntry(ctx, task.entry, childElt.GetName(), task.params.treeContext) if err != nil { return fmt.Errorf("error inserting %s at %s: %w", childElt.GetName(), task.entry.SdcpbPath().ToXPath(false), err) } } - if err := submit(importConfigTask{entry: child, importerElement: childElt, params: task.params}); err != nil { - return err + // need to process Leaflist childs in this goroutine to avois reordering + switch child.GetSchema().GetSchema().(type) { + case *sdcpb.SchemaElem_Leaflist: + err := importConfigTask{entry: child, importerElement: childElt, params: task.params}.Run(ctx, submit) + if err != nil { + return err + } + default: + if err := submit(importConfigTask{entry: child, importerElement: childElt, params: task.params}); err != nil { + return err + } } } return nil @@ -216,47 +232,3 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err return nil } } - -type ImportStats struct { - newEntries atomic.Int64 - updatedEntries atomic.Int64 -} - -func NewImportStats() *ImportStats { - return &ImportStats{} -} - -func (is *ImportStats) String() string { - return fmt.Sprintf("NewEntries: %d, UpdatedEntries: %d", is.newEntries.Load(), is.updatedEntries.Load()) -} - -func (is *ImportStats) Join(i *ImportStats) { - is.newEntries.Add(i.newEntries.Load()) - is.updatedEntries.Add(i.updatedEntries.Load()) -} - -func (is *ImportStats) IncrementNew() { - if is == nil { - return - } - is.newEntries.Add(1) -} - -func (is *ImportStats) IncrementUpdated() { - if is == nil { - return - } - is.updatedEntries.Add(1) -} - -func (is *ImportStats) GetNewCount() int64 { - return is.newEntries.Load() -} - -func (is *ImportStats) GetUpdatedCount() int64 { - return is.updatedEntries.Load() -} - -func (is *ImportStats) Changed() bool { - return is.newEntries.Load() > 0 || is.updatedEntries.Load() > 0 -} diff --git a/pkg/tree/processor_mark_owner_delete.go b/pkg/tree/processor_mark_owner_delete.go index 9de8a279..ffe93abf 100644 --- a/pkg/tree/processor_mark_owner_delete.go +++ b/pkg/tree/processor_mark_owner_delete.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" ) type MarkOwnerDeleteProcessor struct { @@ -76,7 +77,7 @@ func (x ownerDeleteMarkerTask) Run(ctx context.Context, submit func(pool.Task) e x.matches.Append(le) } // Process children recursively - for _, c := range x.e.GetChilds(DescendMethodAll) { + for _, c := range x.e.GetChilds(types.DescendMethodAll) { // Submit may fail if pool is closed or fail-fast error occurred if err := submit(newOwnerDeleteMarkerTask(x.config, c, x.matches)); err != nil { return err diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go index 3beae7b9..d029948c 100644 --- a/pkg/tree/processor_remove_deleted.go +++ b/pkg/tree/processor_remove_deleted.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" ) type RemoveDeletedProcessor struct { @@ -90,7 +91,7 @@ func (t *removeDeletedTask) Run(ctx context.Context, submit func(pool.Task) erro // increment the delete stats count t.config.deleteStatsCount.Add(1) } - if t.e.canDeleteBranch(t.keepDefaults) { + if t.e.CanDeleteBranch(t.keepDefaults) { func() { t.config.zeroLeafEntryElementsLock.Lock() defer t.config.zeroLeafEntryElementsLock.Unlock() @@ -100,7 +101,7 @@ func (t *removeDeletedTask) Run(ctx context.Context, submit func(pool.Task) erro } // Process children recursively - for _, c := range t.e.GetChilds(DescendMethodAll) { + for _, c := range t.e.GetChilds(types.DescendMethodAll) { childTask := newRemoveDeletedTask(t.config, c, t.e.GetSchema().GetContainer() == nil) // Submit may fail if pool is closed or fail-fast error occurred if err := submit(childTask); err != nil { diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go index 921d7b20..b1b462b9 100644 --- a/pkg/tree/processor_reset_flags.go +++ b/pkg/tree/processor_reset_flags.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" ) // ResetFlagsProcessor resets the flags on leaf variant entries @@ -83,7 +84,7 @@ func (t *resetFlagsTask) Run(ctx context.Context, submit func(pool.Task) error) t.config.adjustedFlagsCount.Add(int64(count)) // Process children recursively - for _, c := range t.e.GetChilds(DescendMethodAll) { + for _, c := range t.e.GetChilds(types.DescendMethodAll) { // Submit may fail if pool is closed or fail-fast error occurred if err := submit(newResetFlagsTask(t.config, c)); err != nil { return err diff --git a/pkg/tree/processor_validate.go b/pkg/tree/processor_validate.go index 4d508a47..314d3d91 100644 --- a/pkg/tree/processor_validate.go +++ b/pkg/tree/processor_validate.go @@ -55,9 +55,9 @@ func (t *validateTask) Run(ctx context.Context, submit func(pool.Task) error) er return nil } // validate the mandatory statement on this entry - if t.e.remainsToExist() { + if t.e.RemainsToExist() { t.e.ValidateLevel(ctx, t.parameters.resultChan, t.parameters.stats, t.parameters.vCfg) - for _, c := range t.e.GetChilds(DescendMethodActiveChilds) { + for _, c := range t.e.GetChilds(types.DescendMethodActiveChilds) { submit(newValidateTask(c, t.parameters)) } } diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 5efeb36d..c19ed91d 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -20,7 +20,6 @@ import ( // RootEntry the root of the cache.Update tree type RootEntry struct { *sharedEntryAttributes - explicitDeletes *DeletePathSet } var ( @@ -36,7 +35,6 @@ func NewTreeRoot(ctx context.Context, tc *TreeContext) (*RootEntry, error) { root := &RootEntry{ sharedEntryAttributes: sea, - explicitDeletes: NewDeletePaths(), } err = tc.SetRoot(sea) @@ -62,7 +60,6 @@ func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { result := &RootEntry{ sharedEntryAttributes: se, - explicitDeletes: r.explicitDeletes.DeepCopy(), } err = tc.SetRoot(result.sharedEntryAttributes) @@ -72,10 +69,6 @@ func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { return result, nil } -func (r *RootEntry) RemoveExplicitDeletes(intentName string) *sdcpb.PathSet { - return r.explicitDeletes.RemoveIntentDeletes(intentName) -} - func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAndUpdate, flags *types.UpdateInsertFlags) error { var err error for idx, u := range us { @@ -88,28 +81,21 @@ func (r *RootEntry) AddUpdatesRecursive(ctx context.Context, us []*types.PathAnd return nil } -func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error { - r.treeContext.SetActualOwner(importer.GetName()) - +func (r *RootEntry) ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) { e, err := r.sharedEntryAttributes.getOrCreateChilds(ctx, basePath) if err != nil { - return err + return nil, err } - // store non revertive info - r.treeContext.nonRevertiveInfo[importer.GetName()] = importer.GetNonRevertive() - - // store explicit deletes - r.explicitDeletes.Add(importer.GetName(), importer.GetPriority(), importer.GetDeletes()) - - return e.ImportConfig(ctx, importer, flags, poolFactory) -} - -func (r *RootEntry) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { - r.explicitDeletes.Add(intentName, priority, pathset) + ImportConfigProcessor := NewImportConfigProcessor(importer, flags) + err = ImportConfigProcessor.Run(ctx, e, poolFactory) + if err != nil { + return nil, err + } + return ImportConfigProcessor.GetStats(), nil } func (r *RootEntry) SetNonRevertiveIntent(intentName string, nonRevertive bool) { - r.treeContext.nonRevertiveInfo[intentName] = nonRevertive + r.GetTreeContext().nonRevertiveInfo[intentName] = nonRevertive } func (r *RootEntry) Validate(ctx context.Context, vCfg *config.Validation, taskpoolFactory pool.VirtualPoolFactory) (types.ValidationResults, *types.ValidationStats) { @@ -197,7 +183,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32) (*tree_persist.Inte return nil, err } - explicitDeletes := r.explicitDeletes.GetByIntentName(owner).ToPathSlice() + explicitDeletes := r.treeContext.explicitDeletes.GetByIntentName(owner).ToPathSlice() var rootExportEntry *tree_persist.TreeElement if len(treeExport) != 0 { @@ -253,7 +239,7 @@ func (r *RootEntry) FinishInsertionPhase(ctx context.Context) error { edvs := ExplicitDeleteVisitors{} // apply the explicit deletes - for deletePathPrio := range r.explicitDeletes.Items() { + for deletePathPrio := range r.treeContext.explicitDeletes.Items() { edv := NewExplicitDeleteVisitor(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) for path := range deletePathPrio.PathItems() { diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 4f7d832f..f4b0336e 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -307,7 +307,6 @@ func TestRootEntry_TreeExport(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := &RootEntry{ sharedEntryAttributes: tt.sharedEntryAttributes(), - explicitDeletes: NewDeletePaths(), } got, err := r.TreeExport(tt.args.owner, tt.args.priority) if (err != nil) != tt.wantErr { @@ -405,7 +404,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } @@ -523,7 +522,8 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { } vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = s.ImportConfig(ctx, jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags(), vpf) + ImportConfigProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags()) + err = ImportConfigProcessor.Run(ctx, s, vpf) if err != nil { t.Fatal(err) } @@ -573,11 +573,11 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) if err != nil { t.Fatal(err) } @@ -594,7 +594,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sharedEntryAttributes.go b/pkg/tree/sharedEntryAttributes.go index 0974369a..907dbef3 100644 --- a/pkg/tree/sharedEntryAttributes.go +++ b/pkg/tree/sharedEntryAttributes.go @@ -14,8 +14,6 @@ import ( "unicode/utf8" "github.com/sdcio/data-server/pkg/config" - "github.com/sdcio/data-server/pkg/pool" - "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" logf "github.com/sdcio/logger" @@ -123,7 +121,7 @@ func (s *sharedEntryAttributes) GetRoot() Entry { return s.parent.GetRoot() } -func (s *sharedEntryAttributes) getTreeContext() *TreeContext { +func (s *sharedEntryAttributes) GetTreeContext() *TreeContext { return s.treeContext } @@ -181,7 +179,7 @@ func (s *sharedEntryAttributes) GetDeviations(ctx context.Context, ch chan<- *ty // if s is a presence container but has active childs, it should not be treated as a presence // container, hence the leafvariants should not be processed. For presence container with // childs the TypedValue.empty_val in the presence container is irrelevant. - if s.schema.GetContainer().GetIsPresence() && len(s.GetChilds(DescendMethodActiveChilds)) > 0 { + if s.schema.GetContainer().GetIsPresence() && len(s.GetChilds(types.DescendMethodActiveChilds)) > 0 { evalLeafvariants = false } @@ -191,7 +189,7 @@ func (s *sharedEntryAttributes) GetDeviations(ctx context.Context, ch chan<- *ty } // get all active childs - activeChilds := s.GetChilds(DescendMethodActiveChilds) + activeChilds := s.GetChilds(types.DescendMethodActiveChilds) // iterate through all childs for cName, c := range s.getChildren() { @@ -261,14 +259,14 @@ func (s *sharedEntryAttributes) checkAndCreateKeysAsLeafs(ctx context.Context, i } if !entryExists { // create a new entry - child, err = newEntry(ctx, s, k.Name, s.treeContext) + child, err = NewEntry(ctx, s, k.Name, s.treeContext) if err != nil { return err } } // add the new child entry to s - err = s.addChild(ctx, child) + err = s.AddChild(ctx, child) if err != nil { return err } @@ -358,7 +356,7 @@ func (s *sharedEntryAttributes) GetListChilds() ([]Entry, error) { for level := 0; level < len(keys); level++ { for _, e := range actualEntries { // add all children - for _, c := range e.GetChilds(DescendMethodAll) { + for _, c := range e.GetChilds(types.DescendMethodAll) { newEntries = append(newEntries, c) } } @@ -395,7 +393,7 @@ func (s *sharedEntryAttributes) FilterChilds(keys map[string]string) ([]Entry, e // therefor we need to go through the processEntries List // and collect all the matching childs for _, entry := range processEntries { - childs := entry.GetChilds(DescendMethodAll) + childs := entry.GetChilds(types.DescendMethodAll) matchEntry, childExists := childs[keyVal] // so if such child, that matches the given filter value exists, we append it to the results if childExists { @@ -406,7 +404,7 @@ func (s *sharedEntryAttributes) FilterChilds(keys map[string]string) ([]Entry, e // this is basically the wildcard case, so go through all childs and add them result = []Entry{} for _, entry := range processEntries { - childs := entry.GetChilds(DescendMethodAll) + childs := entry.GetChilds(types.DescendMethodAll) for _, v := range childs { // hence we add all the existing childs to the result list result = append(result, v) @@ -559,7 +557,7 @@ func (s *sharedEntryAttributes) canDelete() bool { } // handle containers - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { canDelete := c.canDelete() if !canDelete { s.cacheCanDelete = utils.BoolPtr(false) @@ -570,7 +568,7 @@ func (s *sharedEntryAttributes) canDelete() bool { return *s.cacheCanDelete } -func (s *sharedEntryAttributes) canDeleteBranch(keepDefault bool) bool { +func (s *sharedEntryAttributes) CanDeleteBranch(keepDefault bool) bool { s.cacheMutex.Lock() defer s.cacheMutex.Unlock() @@ -581,7 +579,7 @@ func (s *sharedEntryAttributes) canDeleteBranch(keepDefault bool) bool { // handle containers for _, c := range s.childs.Items() { - canDelete := c.canDeleteBranch(keepDefault) + canDelete := c.CanDeleteBranch(keepDefault) if !canDelete { return false } @@ -606,7 +604,7 @@ func (s *sharedEntryAttributes) shouldDelete() bool { // but a real delete should only be added if there is at least one shouldDelete() == true shouldDelete := false - activeChilds := s.GetChilds(DescendMethodActiveChilds) + activeChilds := s.GetChilds(types.DescendMethodActiveChilds) // if we have no active childs, we can and should delete. if len(s.choicesResolvers) > 0 && len(activeChilds) == 0 { canDelete = true @@ -642,7 +640,7 @@ func (s *sharedEntryAttributes) shouldDelete() bool { return result } -func (s *sharedEntryAttributes) remainsToExist() bool { +func (s *sharedEntryAttributes) RemainsToExist() bool { // see if we have the value cached s.cacheMutex.Lock() defer s.cacheMutex.Unlock() @@ -653,8 +651,8 @@ func (s *sharedEntryAttributes) remainsToExist() bool { // handle containers childsRemain := false - for _, c := range s.GetChilds(DescendMethodActiveChilds) { - childsRemain = c.remainsToExist() + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { + childsRemain = c.RemainsToExist() if childsRemain { break } @@ -745,8 +743,8 @@ func (s *sharedEntryAttributes) String() string { return s.SdcpbPath().ToXPath(false) } -// addChild add an entry to the list of child entries for the entry. -func (s *sharedEntryAttributes) addChild(ctx context.Context, e Entry) error { +// AddChild add an entry to the list of child entries for the entry. +func (s *sharedEntryAttributes) AddChild(ctx context.Context, e Entry) error { // make sure Entry should not only hold LeafEntries if s.leafVariants.Length() > 0 { // An exception are presence containers @@ -786,7 +784,7 @@ func (s *sharedEntryAttributes) NavigateSdcpbPath(ctx context.Context, path *sdc } return entry.NavigateSdcpbPath(ctx, path.CopyAndRemoveFirstPathElem()) default: - e, exists := s.GetChilds(DescendMethodActiveChilds)[pathElems[0].Name] + e, exists := s.GetChilds(types.DescendMethodActiveChilds)[pathElems[0].Name] if !exists { pth := &sdcpb.Path{Elem: pathElems} e, err = s.tryLoadingDefault(ctx, pth) @@ -860,7 +858,7 @@ func (s *sharedEntryAttributes) DeleteBranch(ctx context.Context, path *sdcpb.Pa // which is, forwarding entry to entry.GetParent() as a last step and depending on the remains // return continuing to perform the delete forther up in the tree // with remains initially set to false, we initially call DeleteSubtree on the referenced entry. - for entry.canDeleteBranch(false) { + for entry.CanDeleteBranch(false) { // forward the entry pointer to the parent // depending on the remains var the DeleteSubtree is again called on that parent entry entry = entry.GetParent() @@ -870,16 +868,16 @@ func (s *sharedEntryAttributes) DeleteBranch(ctx context.Context, path *sdcpb.Pa } // calling DeleteSubtree with the empty string, because it should not delete the owner from the higher level keys, // but what it will also do is delete possibly dangling key elements in the tree - entry.deleteCanDeleteChilds(true) + entry.DeleteCanDeleteChilds(true) } return nil } -func (s *sharedEntryAttributes) deleteCanDeleteChilds(keepDefault bool) { +func (s *sharedEntryAttributes) DeleteCanDeleteChilds(keepDefault bool) { // otherwise check all for childname, child := range s.childs.Items() { - if child.canDeleteBranch(keepDefault) { + if child.CanDeleteBranch(keepDefault) { s.childs.DeleteChild(childname) } } @@ -895,7 +893,7 @@ func (s *sharedEntryAttributes) deleteBranchInternal(ctx context.Context, owner if err != nil { return err } - if child.canDeleteBranch(false) { + if child.CanDeleteBranch(false) { s.childs.DeleteChild(childName) } } @@ -912,7 +910,7 @@ func (s *sharedEntryAttributes) GetHighestPrecedence(result LeafVariantSlice, on } // continue with childs. Childs are part of choices, process only the "active" (highes precedence) childs - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { result = c.GetHighestPrecedence(result, onlyNewOrUpdated, includeDefaults, includeExplicitDelete) } return result @@ -983,7 +981,7 @@ func (s *sharedEntryAttributes) getHighestPrecedenceValueOfBranch(filter Highest // it will multiplex all the different Validations that need to happen func (s *sharedEntryAttributes) ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) { // validate the mandatory statement on this entry - if s.remainsToExist() { + if s.RemainsToExist() { // TODO: Validate Enums if !vCfg.DisabledValidators.Mandatory { s.validateMandatory(ctx, resultChan, stats) @@ -1120,7 +1118,7 @@ func (s *sharedEntryAttributes) validateMinMaxElements(resultChan chan<- *types. ownersSet := map[string]struct{}{} for _, child := range childs { - childAttributes := child.GetChilds(DescendMethodActiveChilds) + childAttributes := child.GetChilds(types.DescendMethodActiveChilds) keyName := contSchema.GetKeys()[0].GetName() if keyAttr, ok := childAttributes[keyName]; ok { highestPrec := keyAttr.GetHighestPrecedence(nil, false, false, false) @@ -1235,7 +1233,7 @@ func (s *sharedEntryAttributes) validatePattern(resultChan chan<- *types.Validat // defined by the schema are present either in the tree or in the index. func (s *sharedEntryAttributes) validateMandatory(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats) { log := logf.FromContext(ctx) - if !s.remainsToExist() { + if !s.RemainsToExist() { return } if s.schema != nil { @@ -1290,9 +1288,9 @@ func (s *sharedEntryAttributes) validateMandatoryWithKeys(ctx context.Context, l // iterate over the attributes make sure any of these exists for _, attr := range attributes { // first check if the mandatory value is set via the intent, e.g. part of the tree already - v, existsInTree = s.GetChilds(DescendMethodActiveChilds)[attr] + v, existsInTree = s.GetChilds(types.DescendMethodActiveChilds)[attr] // if exists and remains to Exist - if existsInTree && v.remainsToExist() { + if existsInTree && v.RemainsToExist() { // set success to true and break the loop success = true break @@ -1313,7 +1311,7 @@ func (s *sharedEntryAttributes) validateMandatoryWithKeys(ctx context.Context, l return } - for _, c := range s.GetChilds(DescendMethodActiveChilds) { + for _, c := range s.GetChilds(types.DescendMethodActiveChilds) { c.validateMandatoryWithKeys(ctx, level-1, attributes, choiceName, resultChan) } } @@ -1366,7 +1364,7 @@ func (s *sharedEntryAttributes) FinishInsertionPhase(ctx context.Context) error // recurse the call to all (active) entries within the tree. // Thereby already using the choiceCaseResolver via filterActiveChoiceCaseChilds() - for _, child := range s.GetChilds(DescendMethodActiveChilds) { + for _, child := range s.GetChilds(types.DescendMethodActiveChilds) { err = child.FinishInsertionPhase(ctx) if err != nil { return err @@ -1432,15 +1430,15 @@ func (s *sharedEntryAttributes) GetChild(name string) (Entry, bool) { return s.childs.GetEntry(name) } -func (s *sharedEntryAttributes) GetChilds(d DescendMethod) EntryMap { +func (s *sharedEntryAttributes) GetChilds(d types.DescendMethod) EntryMap { if s.schema == nil { return s.childs.GetAll() } switch d { - case DescendMethodAll: + case types.DescendMethodAll: return s.childs.GetAll() - case DescendMethodActiveChilds: + case types.DescendMethodActiveChilds: skipAttributesList := s.choicesResolvers.GetSkipElements() // if there are no items that should be skipped, take a shortcut // and simply return all childs straight away @@ -1607,14 +1605,14 @@ func (s *sharedEntryAttributes) getOrCreateChilds(ctx context.Context, path *sdc var current Entry = s for i, pe := range path.Elem { // Step 1: Find or create the child for the path element name - newCurrent, exists := current.GetChilds(DescendMethodAll)[pe.Name] + newCurrent, exists := current.GetChilds(types.DescendMethodAll)[pe.Name] if !exists { var err error - child, err := newEntry(ctx, current, pe.Name, s.treeContext) + child, err := NewEntry(ctx, current, pe.Name, s.treeContext) if err != nil { return nil, err } - if err := current.addChild(ctx, child); err != nil { + if err := current.AddChild(ctx, child); err != nil { return nil, err } newCurrent = child @@ -1630,14 +1628,14 @@ func (s *sharedEntryAttributes) getOrCreateChilds(ctx context.Context, path *sdc // Step 2: For each key, find or create the key child for _, key := range keys { - newCurrent, exists = current.GetChilds(DescendMethodAll)[pe.Key[key]] + newCurrent, exists = current.GetChilds(types.DescendMethodAll)[pe.Key[key]] if !exists { var err error - keyChild, err := newEntry(ctx, current, pe.Key[key], s.treeContext) + keyChild, err := NewEntry(ctx, current, pe.Key[key], s.treeContext) if err != nil { return nil, err } - if err := current.addChild(ctx, keyChild); err != nil { + if err := current.AddChild(ctx, keyChild); err != nil { return nil, err } newCurrent = keyChild @@ -1688,12 +1686,12 @@ func (s *sharedEntryAttributes) addUpdateRecursiveInternal(ctx context.Context, var x Entry = s var exists bool for name := range path.GetElem()[idx].PathElemNames() { - if e, exists = x.GetChilds(DescendMethodAll)[name]; !exists { - newE, err := newEntry(ctx, x, name, s.treeContext) + if e, exists = x.GetChilds(types.DescendMethodAll)[name]; !exists { + newE, err := NewEntry(ctx, x, name, s.treeContext) if err != nil { return nil, err } - err = x.addChild(ctx, newE) + err = x.AddChild(ctx, newE) if err != nil { return nil, err } @@ -1745,7 +1743,8 @@ func (s *sharedEntryAttributes) GetLeafVariantEntries() LeafVariantEntries { return s.leafVariants } -func (s *sharedEntryAttributes) ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error { - processor := NewImportConfigProcessor(importer, insertFlags) - return processor.Run(ctx, s, poolFactory) -} +// func (s *sharedEntryAttributes) ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error { +// processor := NewImportConfigProcessor(importer, insertFlags) + +// return processor.Run(ctx, s, poolFactory) +// } diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index ba27dacd..20bcb6e0 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -94,6 +94,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { name: "just rootEntry", root: func() *RootEntry { tc := NewTreeContext(nil, owner1) + r := &RootEntry{ sharedEntryAttributes: &sharedEntryAttributes{ pathElemName: "__root__", @@ -103,7 +104,6 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { parent: nil, treeContext: tc, }, - explicitDeletes: NewDeletePaths(), } r.leafVariants = newLeafVariants(tc, r.sharedEntryAttributes) return r @@ -147,7 +147,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Error(err) } @@ -209,11 +209,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -247,11 +247,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -345,7 +345,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -407,7 +407,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { for _, elem := range got { elemNames = append(elemNames, elem.PathName()) elemChilds[elem.PathName()] = []string{} - for k := range elem.GetChilds(DescendMethodAll) { + for k := range elem.GetChilds(types.DescendMethodAll) { elemChilds[elem.PathName()] = append(elemChilds[elem.PathName()], k) } } @@ -460,7 +460,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -475,7 +475,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } @@ -656,7 +656,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -779,7 +779,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -920,7 +920,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { } conf1 := config1() - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -977,7 +977,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -1134,7 +1134,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { } vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vpf) + _, err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vpf) if err != nil { t.Error(err) return @@ -1286,7 +1286,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } @@ -1456,7 +1456,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sorter.go b/pkg/tree/sorter.go index 8bbda48b..54a6433d 100644 --- a/pkg/tree/sorter.go +++ b/pkg/tree/sorter.go @@ -1,16 +1,18 @@ package tree +import "github.com/sdcio/data-server/pkg/tree/types" + func getListEntrySortFunc(parent Entry) func(a, b Entry) int { // return the comparison function return func(a, b Entry) int { keys := parent.GetSchemaKeys() var cmpResult int for _, v := range keys { - achild, exists := a.GetChilds(DescendMethodAll)[v] + achild, exists := a.GetChilds(types.DescendMethodAll)[v] if !exists { return 0 } - bchild, exists := b.GetChilds(DescendMethodAll)[v] + bchild, exists := b.GetChilds(types.DescendMethodAll)[v] if !exists { return 0 } diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index 0acc2230..cad1228a 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -4,6 +4,7 @@ import ( "fmt" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) type TreeContext struct { @@ -11,6 +12,7 @@ type TreeContext struct { schemaClient schemaClient.SchemaClientBound actualOwner string nonRevertiveInfo map[string]bool + explicitDeletes *DeletePathSet } func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *TreeContext { @@ -18,6 +20,7 @@ func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *Tree schemaClient: sc, actualOwner: actualOwner, nonRevertiveInfo: map[string]bool{}, + explicitDeletes: NewDeletePaths(), } } @@ -33,9 +36,18 @@ func (t *TreeContext) deepCopy() *TreeContext { m[k] = v } tc.nonRevertiveInfo = m + tc.explicitDeletes = t.explicitDeletes.DeepCopy() return tc } +func (t *TreeContext) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { + t.explicitDeletes.Add(intentName, priority, pathset) +} + +func (t *TreeContext) RemoveExplicitDeletes(intentName string) *sdcpb.PathSet { + return t.explicitDeletes.RemoveIntentDeletes(intentName) +} + func (t *TreeContext) AddNonRevertiveInfo(intent string, nonRevertive bool) { t.nonRevertiveInfo[intent] = nonRevertive } diff --git a/pkg/tree/types/descend_methods.go b/pkg/tree/types/descend_methods.go new file mode 100644 index 00000000..5291ea78 --- /dev/null +++ b/pkg/tree/types/descend_methods.go @@ -0,0 +1,8 @@ +package types + +type DescendMethod int + +const ( + DescendMethodAll DescendMethod = iota + DescendMethodActiveChilds +) diff --git a/pkg/tree/types/import_stats.go b/pkg/tree/types/import_stats.go new file mode 100644 index 00000000..447627dc --- /dev/null +++ b/pkg/tree/types/import_stats.go @@ -0,0 +1,50 @@ +package types + +import ( + "fmt" + "sync/atomic" +) + +type ImportStats struct { + newEntries atomic.Int64 + updatedEntries atomic.Int64 +} + +func NewImportStats() *ImportStats { + return &ImportStats{} +} + +func (is *ImportStats) String() string { + return fmt.Sprintf("NewEntries: %d, UpdatedEntries: %d", is.newEntries.Load(), is.updatedEntries.Load()) +} + +func (is *ImportStats) Join(i *ImportStats) { + is.newEntries.Add(i.newEntries.Load()) + is.updatedEntries.Add(i.updatedEntries.Load()) +} + +func (is *ImportStats) IncrementNew() { + if is == nil { + return + } + is.newEntries.Add(1) +} + +func (is *ImportStats) IncrementUpdated() { + if is == nil { + return + } + is.updatedEntries.Add(1) +} + +func (is *ImportStats) GetNewCount() int64 { + return is.newEntries.Load() +} + +func (is *ImportStats) GetUpdatedCount() int64 { + return is.updatedEntries.Load() +} + +func (is *ImportStats) Changed() bool { + return is.newEntries.Load() > 0 || is.updatedEntries.Load() > 0 +} diff --git a/pkg/tree/validation_entry_leafref.go b/pkg/tree/validation_entry_leafref.go index 3367270f..25a63c91 100644 --- a/pkg/tree/validation_entry_leafref.go +++ b/pkg/tree/validation_entry_leafref.go @@ -61,7 +61,7 @@ func (s *sharedEntryAttributes) BreadthSearch(ctx context.Context, sdcpbPath *sd } // if the entry is will be deleted by the actual operation, skip it. - if !entry.remainsToExist() { + if !entry.RemainsToExist() { continue } // if we're at the final level, no child filtering is needed any more, diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index e56b957a..0621c126 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -239,7 +239,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 0ffabd7b..41053371 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -68,7 +68,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) } @@ -184,7 +184,7 @@ func TestValidate_RangesSigned(t *testing.T) { sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) @@ -319,7 +319,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) // import via importer - err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) diff --git a/pkg/tree/visitor_base.go b/pkg/tree/visitor_base.go index 96772a11..4cf41bc8 100644 --- a/pkg/tree/visitor_base.go +++ b/pkg/tree/visitor_base.go @@ -1,5 +1,7 @@ package tree +import "github.com/sdcio/data-server/pkg/tree/types" + // BaseVisitor abstract base visitor implementation that all the concrete visitory are ment to embed. type BaseVisitor struct{} @@ -7,6 +9,6 @@ func (b *BaseVisitor) Up() { // noop } -func (b *BaseVisitor) DescendMethod() DescendMethod { - return DescendMethodAll +func (b *BaseVisitor) DescendMethod() types.DescendMethod { + return types.DescendMethodAll } diff --git a/pkg/tree/visitor_explicit_delete_test.go b/pkg/tree/visitor_explicit_delete_test.go index b66d3073..42a651ed 100644 --- a/pkg/tree/visitor_explicit_delete_test.go +++ b/pkg/tree/visitor_explicit_delete_test.go @@ -53,7 +53,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) if err != nil { t.Error(err) } @@ -81,12 +81,12 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -133,12 +133,12 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -276,7 +276,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root := tt.root() - root.AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) + root.treeContext.AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) err := root.FinishInsertionPhase(ctx) if err != nil { diff --git a/pkg/tree/xml.go b/pkg/tree/xml.go index f71b4804..e568a8b8 100644 --- a/pkg/tree/xml.go +++ b/pkg/tree/xml.go @@ -7,6 +7,7 @@ import ( "sort" "github.com/beevik/etree" + "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) @@ -42,7 +43,7 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp // if the entry remains so exist, we need to add it to the xml doc overallDoAdd := false - childs := s.GetChilds(DescendMethodActiveChilds) + childs := s.GetChilds(types.DescendMethodActiveChilds) keys := make([]string, 0, len(childs)) for k := range childs { diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index acfb643d..22a626be 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -140,30 +140,30 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem } type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) error + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) } -func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) error { +func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) (*types.ImportStats, error) { jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, SkipValidation: true, }) if err != nil { - return err + return nil, err } var jsonConfAny any err = json.Unmarshal([]byte(jconfStr), &jsonConfAny) if err != nil { - return err + return nil, err } vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vpf) + _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vpf) if err != nil { - return err + return nil, err } - return nil + return nil, nil } // SplitStringSortDiff split the two strings a and b on sep, sort alphabetical and return the diff From 9862b7a21b637223bc43a5ab2f39bcf8f7ba42d9 Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 3 Feb 2026 09:47:02 +0100 Subject: [PATCH 12/17] fix tests --- mocks/mocktreeentry/entry.go | 78 ----------- pkg/datastore/datastore_rpc.go | 2 +- pkg/datastore/intent_rpc.go | 2 +- pkg/datastore/sync.go | 2 +- pkg/datastore/sync_test.go | 8 +- pkg/datastore/transaction_rpc.go | 2 +- pkg/datastore/transaction_rpc_test.go | 2 +- pkg/datastore/tree_operation_test.go | 2 +- .../tree_operation_validation_test.go | 2 +- pkg/tree/entry.go | 8 +- pkg/tree/entry_test.go | 34 ++--- .../proto/proto_tree_importer_test.go | 4 +- .../importer/xml/xml_tree_importer_test.go | 2 +- pkg/tree/json_test.go | 4 +- pkg/tree/processor_blame_config_test.go | 6 +- pkg/tree/processor_explicit_delete.go | 121 +++++++++++++++++ ...t.go => processor_explicit_delete_test.go} | 8 +- pkg/tree/processor_importer.go | 13 +- pkg/tree/processor_reset_flags_test.go | 2 +- pkg/tree/root_entry.go | 22 +-- pkg/tree/root_entry_test.go | 12 +- pkg/tree/sharedEntryAttributes.go | 20 --- pkg/tree/sharedEntryAttributes_test.go | 32 ++--- pkg/tree/tree_context.go | 10 +- pkg/tree/validation_entry_leafref_test.go | 2 +- pkg/tree/validation_range_test.go | 6 +- pkg/tree/visitor_base.go | 14 -- pkg/tree/visitor_explicit_delete.go | 125 ------------------ pkg/tree/xml_test.go | 2 +- pkg/utils/testhelper/utils.go | 19 ++- 30 files changed, 237 insertions(+), 329 deletions(-) create mode 100644 pkg/tree/processor_explicit_delete.go rename pkg/tree/{visitor_explicit_delete_test.go => processor_explicit_delete_test.go} (96%) delete mode 100644 pkg/tree/visitor_base.go delete mode 100644 pkg/tree/visitor_explicit_delete.go diff --git a/mocks/mocktreeentry/entry.go b/mocks/mocktreeentry/entry.go index 543cd674..ff0b493b 100644 --- a/mocks/mocktreeentry/entry.go +++ b/mocks/mocktreeentry/entry.go @@ -586,20 +586,6 @@ func (mr *MockEntryMockRecorder) ValidateLevel(ctx, resultChan, stats, vCfg any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateLevel", reflect.TypeOf((*MockEntry)(nil).ValidateLevel), ctx, resultChan, stats, vCfg) } -// Walk mocks base method. -func (m *MockEntry) Walk(ctx context.Context, v tree.EntryVisitor) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Walk", ctx, v) - ret0, _ := ret[0].(error) - return ret0 -} - -// Walk indicates an expected call of Walk. -func (mr *MockEntryMockRecorder) Walk(ctx, v any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockEntry)(nil).Walk), ctx, v) -} - // addUpdateRecursiveInternal mocks base method. func (m *MockEntry) addUpdateRecursiveInternal(ctx context.Context, path *sdcpb.Path, idx int, u *types.Update, flags *types.UpdateInsertFlags) (tree.Entry, error) { m.ctrl.T.Helper() @@ -741,70 +727,6 @@ func (mr *MockEntryMockRecorder) validateMandatoryWithKeys(ctx, level, attribute return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "validateMandatoryWithKeys", reflect.TypeOf((*MockEntry)(nil).validateMandatoryWithKeys), ctx, level, attributes, choiceName, resultChan) } -// MockEntryVisitor is a mock of EntryVisitor interface. -type MockEntryVisitor struct { - ctrl *gomock.Controller - recorder *MockEntryVisitorMockRecorder - isgomock struct{} -} - -// MockEntryVisitorMockRecorder is the mock recorder for MockEntryVisitor. -type MockEntryVisitorMockRecorder struct { - mock *MockEntryVisitor -} - -// NewMockEntryVisitor creates a new mock instance. -func NewMockEntryVisitor(ctrl *gomock.Controller) *MockEntryVisitor { - mock := &MockEntryVisitor{ctrl: ctrl} - mock.recorder = &MockEntryVisitorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockEntryVisitor) EXPECT() *MockEntryVisitorMockRecorder { - return m.recorder -} - -// DescendMethod mocks base method. -func (m *MockEntryVisitor) DescendMethod() types.DescendMethod { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescendMethod") - ret0, _ := ret[0].(types.DescendMethod) - return ret0 -} - -// DescendMethod indicates an expected call of DescendMethod. -func (mr *MockEntryVisitorMockRecorder) DescendMethod() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescendMethod", reflect.TypeOf((*MockEntryVisitor)(nil).DescendMethod)) -} - -// Up mocks base method. -func (m *MockEntryVisitor) Up() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Up") -} - -// Up indicates an expected call of Up. -func (mr *MockEntryVisitorMockRecorder) Up() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Up", reflect.TypeOf((*MockEntryVisitor)(nil).Up)) -} - -// Visit mocks base method. -func (m *MockEntryVisitor) Visit(ctx context.Context, e tree.Entry) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Visit", ctx, e) - ret0, _ := ret[0].(error) - return ret0 -} - -// Visit indicates an expected call of Visit. -func (mr *MockEntryVisitorMockRecorder) Visit(ctx, e any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Visit", reflect.TypeOf((*MockEntryVisitor)(nil).Visit), ctx, e) -} - // MockLeafVariantEntry is a mock of LeafVariantEntry interface. type MockLeafVariantEntry struct { ctrl *gomock.Controller diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index bb457be2..722e2904 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -93,7 +93,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca ) scb := schemaClient.NewSchemaClientBound(c.Schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { cancel() diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index dd9db188..bb112e21 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -66,7 +66,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } // otherwise consult cache - root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, intentName)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, intentName, d.taskPool)) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 17d1a0ec..57c243d5 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -99,7 +99,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } func (d *Datastore) NewEmptyTree(ctx context.Context) (*tree.RootEntry, error) { - tc := tree.NewTreeContext(d.schemaClient, tree.RunningIntentName) + tc := tree.NewTreeContext(d.schemaClient, tree.RunningIntentName, d.taskPool) newTree, err := tree.NewTreeRoot(ctx, tc) if err != nil { return nil, err diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index 1129d047..1ba43116 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -54,7 +54,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -160,7 +160,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -264,7 +264,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -390,7 +390,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) resultRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 0e76b97b..577425a7 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -72,7 +72,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa ctx = logf.IntoContext(ctx, log) // create a new TreeContext - tc := tree.NewTreeContext(d.schemaClient, d.Name()) + tc := tree.NewTreeContext(d.schemaClient, d.Name(), d.taskPool) // create a new TreeRoot to collect validate and hand to SBI.Set() root, err := tree.NewTreeRoot(ctx, tc) diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go index c25adadd..363698b5 100644 --- a/pkg/datastore/transaction_rpc_test.go +++ b/pkg/datastore/transaction_rpc_test.go @@ -140,7 +140,7 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { sbi.EXPECT().Set(gomock.Any(), gomock.Any()).Return(&sdcpb.SetDataResponse{}, nil).AnyTimes() // Setup SyncTree with Running Config - tc := tree.NewTreeContext(scb, tree.RunningIntentName) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Fatalf("failed to create sync tree root: %v", err) diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 0ecd35bd..c8574ab6 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1471,7 +1471,7 @@ func TestDatastore_populateTree(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tt.intentName) + tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index 14400854..aec0b17b 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -187,7 +187,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - tc := tree.NewTreeContext(scb, tt.intentName) + tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 6efe1ad8..080cf504 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -65,7 +65,7 @@ type Entry interface { // GetDeletes returns the cache-updates that are not updated, have no lower priority value left and hence should be deleted completely GetDeletes(entries []types.DeleteEntry, aggregatePaths bool) ([]types.DeleteEntry, error) // Walk takes the EntryVisitor and applies it to every Entry in the tree - Walk(ctx context.Context, v EntryVisitor) error + // Walk(ctx context.Context, v EntryVisitor) error // Validate kicks off validation ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) // validateMandatory the Mandatory schema field @@ -148,12 +148,6 @@ type Entry interface { GetTreeContext() *TreeContext } -type EntryVisitor interface { - DescendMethod() types.DescendMethod - Visit(ctx context.Context, e Entry) error - Up() -} - type LeafVariantEntry interface { MarkDelete(onlyIntended bool) *LeafEntry GetEntry() Entry diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index b57b9c2e..d2e6b0f1 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -66,7 +66,7 @@ func Test_Entry(t *testing.T) { u2 := types.NewUpdate(nil, desc, int32(99), "me", int64(444)) u3 := types.NewUpdate(nil, desc, int32(98), "me", int64(88)) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -174,7 +174,7 @@ func Test_Entry_One(t *testing.T) { u3 := types.NewUpdate(nil, desc3, prio50, owner2, ts1) u3_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner2, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -300,7 +300,7 @@ func Test_Entry_Two(t *testing.T) { u1 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) u1_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -495,7 +495,7 @@ func Test_Entry_Three(t *testing.T) { u3r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) u4r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -784,7 +784,7 @@ func Test_Entry_Four(t *testing.T) { } // u2o2_1 := types.NewUpdate(p2o2_1, testhelper.GetUIntTvProto(11), prio55, owner2, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -956,7 +956,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1009,7 +1009,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1058,7 +1058,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Four", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1185,7 +1185,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } u6 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1480,7 +1480,7 @@ func Test_Schema_Population(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1535,7 +1535,7 @@ func Test_sharedEntryAttributes_SdcpbPath(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo") + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1666,7 +1666,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1705,7 +1705,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1802,7 +1802,7 @@ func Test_Validation_Deref(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1858,7 +1858,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Valid", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1936,7 +1936,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_Owner", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1980,7 +1980,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_NextHop", func(t *testing.T) { - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index b646292a..93a882e9 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -114,7 +114,7 @@ func TestProtoTreeImporter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tc := tree.NewTreeContext(scb, "test") + tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -149,7 +149,7 @@ func TestProtoTreeImporter(t *testing.T) { fmt.Println(protoIntent.PrettyString(" ")) - tcNew := tree.NewTreeContext(scb, "test") + tcNew := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) rootNew, err := tree.NewTreeRoot(ctx, tcNew) if err != nil { t.Error(err) diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index a2ae86e3..227aa820 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -79,7 +79,7 @@ func TestXmlTreeImporter(t *testing.T) { } ctx := context.Background() - tc := tree.NewTreeContext(scb, "test") + tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/tree/json_test.go b/pkg/tree/json_test.go index 2f7903ea..c62a6e95 100644 --- a/pkg/tree/json_test.go +++ b/pkg/tree/json_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "runtime" "testing" "github.com/google/go-cmp/cmp" "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils" "github.com/sdcio/data-server/pkg/utils/testhelper" @@ -378,7 +380,7 @@ func TestToJsonTable(t *testing.T) { ctx := context.Background() - tc := NewTreeContext(scb, owner) + tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index f34e5f24..d9500575 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -39,7 +39,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -104,7 +104,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/processor_explicit_delete.go b/pkg/tree/processor_explicit_delete.go new file mode 100644 index 00000000..a8399761 --- /dev/null +++ b/pkg/tree/processor_explicit_delete.go @@ -0,0 +1,121 @@ +package tree + +import ( + "context" + "sync" + + "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree/types" + "github.com/sdcio/data-server/pkg/utils" +) + +type ExplicitDeleteProcessor struct { + params *ExplicitDeleteTaskParameters +} + +func NewExplicitDeleteProcessor(params *ExplicitDeleteTaskParameters) *ExplicitDeleteProcessor { + return &ExplicitDeleteProcessor{ + params: params, + } +} + +// GetExplicitDeleteCreationCount returns the amount of all the explicitDelete LeafVariants that where created. +func (edp *ExplicitDeleteProcessor) GetExplicitDeleteCreationCount() int { + return len(edp.params.relatedLeafVariants) +} + +// GetCreatedExplicitDeleteLeafEntries returns all the explicitDelete LeafVariants that where created. +func (edp *ExplicitDeleteProcessor) GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice { + return edp.params.relatedLeafVariants +} + +type ExplicitDeleteTaskParameters struct { + owner string + priority int32 + relatedLeafVariants []*LeafEntry + rlvMutex *sync.Mutex +} + +func NewExplicitDeleteTaskParameters(owner string, priority int32) *ExplicitDeleteTaskParameters { + return &ExplicitDeleteTaskParameters{ + priority: priority, + owner: owner, + relatedLeafVariants: []*LeafEntry{}, + rlvMutex: &sync.Mutex{}, + } +} + +func (p *ExplicitDeleteProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { + taskpool := poolFactory.NewVirtualPool(pool.VirtualTolerant) + err := taskpool.Submit(newExplicitDeleteTask(e, p.params)) + taskpool.CloseAndWait() + return err +} + +type explicitDeleteTask struct { + entry Entry + params *ExplicitDeleteTaskParameters +} + +func newExplicitDeleteTask(entry Entry, params *ExplicitDeleteTaskParameters) *explicitDeleteTask { + return &explicitDeleteTask{ + entry: entry, + params: params, + } +} + +func (t *explicitDeleteTask) Run(ctx context.Context, submit func(pool.Task) error) error { + if !t.entry.HoldsLeafvariants() { + return nil + } + le := t.entry.GetLeafVariantEntries().GetByOwner(t.params.owner) + if le != nil { + le.MarkExpliciteDelete() + } else { + le = t.entry.GetLeafVariantEntries().AddExplicitDeleteEntry(t.params.owner, t.params.priority) + } + t.params.rlvMutex.Lock() + t.params.relatedLeafVariants = append(t.params.relatedLeafVariants, le) + t.params.rlvMutex.Unlock() + + // trigger the execution on all childs + for _, c := range t.entry.GetChilds(types.DescendMethodAll) { + err := submit(newExplicitDeleteTask(c, t.params)) + if err != nil { + return err + } + } + + return nil +} + +// Stats structs +type ExplicitDeleteProcessorStat interface { + GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice + GetExplicitDeleteCreationCount() int +} + +type ExplicitDeleteProcessorStatCollection map[string]ExplicitDeleteProcessorStat + +func (e ExplicitDeleteProcessorStatCollection) Stats() map[string]int { + return utils.MapApplyFuncToMap(e, func(k string, v ExplicitDeleteProcessorStat) int { + return v.GetExplicitDeleteCreationCount() + }) +} + +func (e ExplicitDeleteProcessorStatCollection) Count() int { + count := 0 + for _, stat := range e { + count += stat.GetExplicitDeleteCreationCount() + } + return count +} + +func (e ExplicitDeleteProcessorStatCollection) ContainsEntries() bool { + for _, stat := range e { + if stat.GetExplicitDeleteCreationCount() > 0 { + return true + } + } + return false +} diff --git a/pkg/tree/visitor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go similarity index 96% rename from pkg/tree/visitor_explicit_delete_test.go rename to pkg/tree/processor_explicit_delete_test.go index 42a651ed..8aaa3fe0 100644 --- a/pkg/tree/visitor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -2,11 +2,13 @@ package tree import ( "context" + "runtime" "strings" "testing" "github.com/openconfig/ygot/ygot" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" "github.com/sdcio/data-server/pkg/tree/types" "github.com/sdcio/data-server/pkg/utils/testhelper" sdcio_schema "github.com/sdcio/data-server/tests/sdcioygot" @@ -46,7 +48,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -74,7 +76,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -126,7 +128,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/processor_importer.go b/pkg/tree/processor_importer.go index 3c0c7a06..9b459976 100644 --- a/pkg/tree/processor_importer.go +++ b/pkg/tree/processor_importer.go @@ -7,7 +7,8 @@ import ( "sync" "github.com/sdcio/data-server/pkg/pool" - "github.com/sdcio/data-server/pkg/tree/importer" + + treeimporter "github.com/sdcio/data-server/pkg/tree/importer" "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/protobuf/types/known/emptypb" @@ -15,7 +16,7 @@ import ( type importConfigTask struct { entry Entry - importerElement importer.ImportConfigAdapterElement + importerElement treeimporter.ImportConfigAdapterElement params *ImportConfigProcessorParams } @@ -28,7 +29,7 @@ type ImportConfigProcessorParams struct { stats *types.ImportStats } -func NewImportConfigProcessorParams( +func NewParameters( intentName string, intentPrio int32, insertFlags *types.UpdateInsertFlags, @@ -47,12 +48,12 @@ func NewImportConfigProcessorParams( } type ImportConfigProcessor struct { - importer importer.ImportConfigAdapter + importer treeimporter.ImportConfigAdapter insertFlags *types.UpdateInsertFlags stats *types.ImportStats } -func NewImportConfigProcessor(importer importer.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { +func NewImportConfigProcessor(importer treeimporter.ImportConfigAdapter, insertFlags *types.UpdateInsertFlags) *ImportConfigProcessor { return &ImportConfigProcessor{ importer: importer, insertFlags: insertFlags, @@ -79,7 +80,7 @@ func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory po t := importConfigTask{ entry: e, importerElement: p.importer, - params: NewImportConfigProcessorParams(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.GetTreeContext(), &sync.Map{}, p.stats), + params: NewParameters(p.importer.GetName(), p.importer.GetPriority(), p.insertFlags, e.GetTreeContext(), &sync.Map{}, p.stats), } if err := workerPool.Submit(t); err != nil { diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go index 57488969..642de8dd 100644 --- a/pkg/tree/processor_reset_flags_test.go +++ b/pkg/tree/processor_reset_flags_test.go @@ -42,7 +42,7 @@ func TestResetFlagsProcessorRun(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, RunningIntentName) + tc := NewTreeContext(scb, RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index c19ed91d..80582fa0 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -236,31 +236,35 @@ func (r *RootEntry) DeleteBranchPaths(ctx context.Context, deletes types.DeleteE func (r *RootEntry) FinishInsertionPhase(ctx context.Context) error { log := logf.FromContext(ctx) - edvs := ExplicitDeleteVisitors{} + edpsc := ExplicitDeleteProcessorStatCollection{} // apply the explicit deletes for deletePathPrio := range r.treeContext.explicitDeletes.Items() { - edv := NewExplicitDeleteVisitor(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) + + params := NewExplicitDeleteTaskParameters(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) for path := range deletePathPrio.PathItems() { + // set the priority // navigate to the stated path entry, err := r.NavigateSdcpbPath(ctx, path) if err != nil { log.Error(nil, "Applying explicit delete - path not found, skipping", "severity", "WARN", "path", path.ToXPath(false)) } - - // walk the whole branch adding the explicit delete leafvariant - err = entry.Walk(ctx, edv) + edp := NewExplicitDeleteProcessor(params) + err = edp.Run(ctx, entry, r.GetTreeContext().GetPoolFactory()) if err != nil { return err } - edvs[deletePathPrio.GetOwner()] = edv + edpsc[deletePathPrio.GetOwner()] = edp } } - log.V(logf.VDebug).Info("ExplicitDeletes added", "explicit-deletes", utils.MapToString(edvs.Stats(), ", ", func(k string, v int) string { - return fmt.Sprintf("%s=%d", k, v) - })) + // conditional logging + if edpsc.ContainsEntries() { + log.V(logf.VDebug).Info("ExplicitDeletes added", "explicit-deletes", utils.MapToString(edpsc.Stats(), ", ", func(k string, v int) string { + return fmt.Sprintf("%s=%d", k, v) + })) + } return r.sharedEntryAttributes.FinishInsertionPhase(ctx) } diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index f4b0336e..206eafd7 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -26,7 +26,7 @@ import ( func TestRootEntry_TreeExport(t *testing.T) { owner1 := "owner1" owner2 := "owner2" - tc := NewTreeContext(nil, owner1) + tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.NumCPU())) type args struct { owner string @@ -397,7 +397,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -434,7 +434,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) type fields struct { sharedEntryAttributes func(t *testing.T) *sharedEntryAttributes @@ -568,7 +568,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { { name: "One", rootEntry: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -589,7 +589,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { }, owner: owner1, want: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -611,7 +611,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { root := tt.rootEntry(t) got := root.GetUpdatesForOwner(tt.owner).ToPathAndUpdateSlice() - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) resultRoot, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/sharedEntryAttributes.go b/pkg/tree/sharedEntryAttributes.go index 907dbef3..6ea355b1 100644 --- a/pkg/tree/sharedEntryAttributes.go +++ b/pkg/tree/sharedEntryAttributes.go @@ -444,26 +444,6 @@ func (s *sharedEntryAttributes) GetLevel() int { return l } -// Walk takes the EntryVisitor and applies it to every Entry in the tree -func (s *sharedEntryAttributes) Walk(ctx context.Context, v EntryVisitor) error { - - // execute the function locally - err := v.Visit(ctx, s) - if err != nil { - return err - } - - // trigger the execution on all childs - for _, c := range s.GetChilds(v.DescendMethod()) { - err := c.Walk(ctx, v) - if err != nil { - return err - } - } - v.Up() - return nil -} - func (s *sharedEntryAttributes) HoldsLeafvariants() bool { switch x := s.schema.GetSchema().(type) { case *sdcpb.SchemaElem_Container: diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index 20bcb6e0..4ea0b7ba 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -37,7 +37,7 @@ func Test_sharedEntryAttributes_checkAndCreateKeysAsLeafs(t *testing.T) { ctx := context.Background() scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1") + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -93,7 +93,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { { name: "just rootEntry", root: func() *RootEntry { - tc := NewTreeContext(nil, owner1) + tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.NumCPU())) r := &RootEntry{ sharedEntryAttributes: &sharedEntryAttributes{ @@ -123,7 +123,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -204,7 +204,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -242,7 +242,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -339,7 +339,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -453,7 +453,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -632,7 +632,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -755,7 +755,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -867,7 +867,7 @@ func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -913,7 +913,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -945,7 +945,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1086,7 +1086,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1127,7 +1127,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println("\nTreeExport:") fmt.Println(string(persistByte)) - tcNew := NewTreeContext(scb, owner1) + tcNew := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) newRoot, err := NewTreeRoot(ctx, tcNew) if err != nil { t.Fatal(err) @@ -1262,7 +1262,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1432,7 +1432,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index cad1228a..9da8ab59 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -4,6 +4,7 @@ import ( "fmt" schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" + "github.com/sdcio/data-server/pkg/pool" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) @@ -13,14 +14,16 @@ type TreeContext struct { actualOwner string nonRevertiveInfo map[string]bool explicitDeletes *DeletePathSet + poolFactory pool.VirtualPoolFactory } -func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *TreeContext { +func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string, poolFactory pool.VirtualPoolFactory) *TreeContext { return &TreeContext{ schemaClient: sc, actualOwner: actualOwner, nonRevertiveInfo: map[string]bool{}, explicitDeletes: NewDeletePaths(), + poolFactory: poolFactory, } } @@ -28,6 +31,7 @@ func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string) *Tree func (t *TreeContext) deepCopy() *TreeContext { tc := &TreeContext{ schemaClient: t.schemaClient, + poolFactory: t.poolFactory, } // deepcopy nonRevertiveInfo @@ -40,6 +44,10 @@ func (t *TreeContext) deepCopy() *TreeContext { return tc } +func (t *TreeContext) GetPoolFactory() pool.VirtualPoolFactory { + return t.poolFactory +} + func (t *TreeContext) AddExplicitDeletes(intentName string, priority int32, pathset *sdcpb.PathSet) { t.explicitDeletes.Add(intentName, priority, pathset) } diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 0621c126..5c8fc947 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -215,7 +215,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 41053371..8716069a 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -27,7 +27,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) @@ -157,7 +157,7 @@ func TestValidate_RangesSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -293,7 +293,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1") + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) // the tree root root, err := NewTreeRoot(ctx, tc) diff --git a/pkg/tree/visitor_base.go b/pkg/tree/visitor_base.go deleted file mode 100644 index 4cf41bc8..00000000 --- a/pkg/tree/visitor_base.go +++ /dev/null @@ -1,14 +0,0 @@ -package tree - -import "github.com/sdcio/data-server/pkg/tree/types" - -// BaseVisitor abstract base visitor implementation that all the concrete visitory are ment to embed. -type BaseVisitor struct{} - -func (b *BaseVisitor) Up() { - // noop -} - -func (b *BaseVisitor) DescendMethod() types.DescendMethod { - return types.DescendMethodAll -} diff --git a/pkg/tree/visitor_explicit_delete.go b/pkg/tree/visitor_explicit_delete.go deleted file mode 100644 index 1c1f5c83..00000000 --- a/pkg/tree/visitor_explicit_delete.go +++ /dev/null @@ -1,125 +0,0 @@ -package tree - -import ( - "context" - "sync" - - "github.com/sdcio/data-server/pkg/utils" -) - -type ExplicitDeleteVisitor struct { - BaseVisitor - owner string - priority int32 - - // created entries for further stat calculation - relatedLeafVariants LeafVariantSlice - rlvMutex *sync.Mutex -} - -var _ EntryVisitor = (*ExplicitDeleteVisitor)(nil) - -func NewExplicitDeleteVisitor(owner string, priority int32) *ExplicitDeleteVisitor { - return &ExplicitDeleteVisitor{ - priority: priority, - owner: owner, - relatedLeafVariants: []*LeafEntry{}, - rlvMutex: &sync.Mutex{}, - } -} - -func (edv *ExplicitDeleteVisitor) Visit(ctx context.Context, e Entry) error { - if !e.HoldsLeafvariants() { - return nil - } - le := e.GetLeafVariantEntries().GetByOwner(edv.owner) - if le != nil { - le.MarkExpliciteDelete() - } else { - le = e.GetLeafVariantEntries().AddExplicitDeleteEntry(edv.owner, edv.priority) - } - edv.rlvMutex.Lock() - edv.relatedLeafVariants = append(edv.relatedLeafVariants, le) - edv.rlvMutex.Unlock() - return nil -} - -// GetExplicitDeleteCreationCount returns the amount of all the explicitDelete LeafVariants that where created. -func (edv *ExplicitDeleteVisitor) GetExplicitDeleteCreationCount() int { - return len(edv.relatedLeafVariants) -} - -// GetCreatedExplicitDeleteLeafEntries returns all the explicitDelete LeafVariants that where created. -func (edv *ExplicitDeleteVisitor) GetCreatedExplicitDeleteLeafEntries() LeafVariantSlice { - return edv.relatedLeafVariants -} - -// ExplicitDeleteVisitors map of *ExplicitDeleteVisitor indexed by the intent name -type ExplicitDeleteVisitors map[string]*ExplicitDeleteVisitor - -func (e ExplicitDeleteVisitors) Stats() map[string]int { - return utils.MapApplyFuncToMap(e, func(k string, v *ExplicitDeleteVisitor) int { - return v.GetExplicitDeleteCreationCount() - }) -} - -// type ExplicitDeleteTask struct { -// e Entry -// owner string -// priority int32 -// } - -// func NewExplicitDeleteTask(e Entry, owner string, priority int32) *ExplicitDeleteTask { -// return &ExplicitDeleteTask{ -// e: e, -// owner: owner, -// priority: priority, -// } -// } - -// func (edt *ExplicitDeleteTask) Run(ctx context.Context, submit func(pool.Task) error) error { - -// if !edt.e.HoldsLeafvariants() { -// return nil -// } -// le := edt.e.GetLeafVariantEntries().GetByOwner(edt.owner) -// if le != nil { -// le.MarkExpliciteDelete() -// } else { -// le = edt.e.GetLeafVariantEntries().AddExplicitDeleteEntry(edt.owner, edt.priority) -// } - -// for _, c := range edt.e.GetChilds(DescendMethodAll) { -// deleteTask := NewExplicitDeleteTask(c, edt.owner, edt.priority) -// err := submit(deleteTask) -// if err != nil { -// return err -// } -// } -// return nil - -// } - -// func ExplicitDelete(ctx context.Context, e Entry, dps *DeletePathSet, pool pool.VirtualPoolI) error { -// for dp := range dps.Items() { -// for pi := range dp.PathItems() { -// edp, err := e.NavigateSdcpbPath(ctx, pi) -// if err != nil { -// return err -// } - -// deleteTask := NewExplicitDeleteTask(edp, dp.GetOwner(), dp.GetPrio()) - -// err = pool.Submit(deleteTask) -// if err != nil { -// return err -// } -// } -// } - -// // close pool for additional external submission -// pool.CloseForSubmit() -// // wait for the pool to run dry -// pool.Wait() -// return nil -// } diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index 6ffe57d3..c142d81e 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -574,7 +574,7 @@ func TestToXMLTable(t *testing.T) { converter := utils.NewConverter(scb) - tc := NewTreeContext(scb, owner) + tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 22a626be..72160cb7 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -13,6 +13,7 @@ import ( "github.com/openconfig/ygot/ygot" "github.com/sdcio/data-server/mocks/mockschemaclientbound" "github.com/sdcio/data-server/pkg/pool" + "github.com/sdcio/data-server/pkg/tree" "github.com/sdcio/data-server/pkg/tree/importer" jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" @@ -143,6 +144,15 @@ type RootTreeImport interface { ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) } +type ImportStatsInterface interface { + Changed() bool + GetNewCount() int64 + GetUpdatedCount() int64 + IncrementNew() + IncrementUpdated() + String() string +} + func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) (*types.ImportStats, error) { jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, @@ -158,12 +168,15 @@ func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root Root return nil, err } - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags, vpf) + stp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + + importProcessor := tree.NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) + err = importProcessor.Run(ctx, root, stp) + if err != nil { return nil, err } - return nil, nil + return importProcessor.GetStats(), nil } // SplitStringSortDiff split the two strings a and b on sep, sort alphabetical and return the diff From c0a745c059adb5656ca5fd8b7f5423e938389488 Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 3 Feb 2026 09:47:02 +0100 Subject: [PATCH 13/17] fix tests --- pkg/tree/root_entry_test.go | 2 +- pkg/tree/utils.go | 45 +++++++++++++++++++++++++++++++++++ pkg/utils/testhelper/utils.go | 37 ---------------------------- 3 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 pkg/tree/utils.go diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 206eafd7..5b983e9c 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -404,7 +404,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, tt.re(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/utils.go b/pkg/tree/utils.go new file mode 100644 index 00000000..ce938649 --- /dev/null +++ b/pkg/tree/utils.go @@ -0,0 +1,45 @@ +package tree + +import ( + "context" + "encoding/json" + "runtime" + + "github.com/openconfig/ygot/ygot" + "github.com/sdcio/data-server/pkg/pool" + + "github.com/sdcio/data-server/pkg/tree/importer" + jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" + "github.com/sdcio/data-server/pkg/tree/types" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +type RootTreeImport interface { + ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) +} + +func loadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root *RootEntry, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) (*types.ImportStats, error) { + jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ + Format: ygot.RFC7951, + SkipValidation: true, + }) + if err != nil { + return nil, err + } + + var jsonConfAny any + err = json.Unmarshal([]byte(jconfStr), &jsonConfAny) + if err != nil { + return nil, err + } + + stp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + + importProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) + err = importProcessor.Run(ctx, root.sharedEntryAttributes, stp) + + if err != nil { + return nil, err + } + return importProcessor.GetStats(), nil +} diff --git a/pkg/utils/testhelper/utils.go b/pkg/utils/testhelper/utils.go index 72160cb7..8d170367 100644 --- a/pkg/utils/testhelper/utils.go +++ b/pkg/utils/testhelper/utils.go @@ -2,20 +2,13 @@ package testhelper import ( "context" - "encoding/json" "fmt" - "runtime" "slices" "strings" "testing" "github.com/google/go-cmp/cmp" - "github.com/openconfig/ygot/ygot" "github.com/sdcio/data-server/mocks/mockschemaclientbound" - "github.com/sdcio/data-server/pkg/pool" - "github.com/sdcio/data-server/pkg/tree" - "github.com/sdcio/data-server/pkg/tree/importer" - jsonImporter "github.com/sdcio/data-server/pkg/tree/importer/json" "github.com/sdcio/data-server/pkg/tree/types" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "go.uber.org/mock/gomock" @@ -140,10 +133,6 @@ func GetSchemaClientBound(t *testing.T, mockCtrl *gomock.Controller) (*mockschem return mockscb, nil } -type RootTreeImport interface { - ImportConfig(ctx context.Context, basePath *sdcpb.Path, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) -} - type ImportStatsInterface interface { Changed() bool GetNewCount() int64 @@ -153,32 +142,6 @@ type ImportStatsInterface interface { String() string } -func LoadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root RootTreeImport, owner string, prio int32, nonRevertive bool, flags *types.UpdateInsertFlags) (*types.ImportStats, error) { - jconfStr, err := ygot.EmitJSON(gs, &ygot.EmitJSONConfig{ - Format: ygot.RFC7951, - SkipValidation: true, - }) - if err != nil { - return nil, err - } - - var jsonConfAny any - err = json.Unmarshal([]byte(jconfStr), &jsonConfAny) - if err != nil { - return nil, err - } - - stp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) - - importProcessor := tree.NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) - err = importProcessor.Run(ctx, root, stp) - - if err != nil { - return nil, err - } - return importProcessor.GetStats(), nil -} - // SplitStringSortDiff split the two strings a and b on sep, sort alphabetical and return the diff func SplitStringSortDiff(a, b string, sep string) string { From 9d3371eee2ebb5e5d9ed15c71dc5992157ac2768 Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 3 Feb 2026 13:24:40 +0100 Subject: [PATCH 14/17] update mod & fix --- go.mod | 4 +--- go.sum | 2 ++ pkg/tree/processor_blame_config_test.go | 10 +++++----- pkg/tree/processor_explicit_delete_test.go | 12 ++++++------ pkg/tree/root_entry_test.go | 6 +++--- pkg/tree/sharedEntryAttributes_test.go | 18 +++++++++--------- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 4013ff08..d7af0726 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.24.0 replace github.com/openconfig/goyang v1.6.0 => github.com/sdcio/goyang v1.6.2-2 -replace github.com/sdcio/sdc-protos => /home/mava/projects/sdc-protos - require ( github.com/AlekSi/pointer v1.2.0 github.com/beevik/etree v1.6.0 @@ -26,7 +24,7 @@ require ( github.com/sdcio/cache v0.0.38 github.com/sdcio/logger v0.0.3 github.com/sdcio/schema-server v0.0.33 - github.com/sdcio/sdc-protos v0.0.47 + github.com/sdcio/sdc-protos v0.0.50 github.com/sdcio/yang-parser v0.0.12 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 diff --git a/go.sum b/go.sum index efc8a2b1..6ea519b2 100644 --- a/go.sum +++ b/go.sum @@ -189,6 +189,8 @@ github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= github.com/sdcio/logger v0.0.3/go.mod h1:yWaOxK/G6vszjg8tKZiMqiEjlZouHsjFME4zSk+SAEA= github.com/sdcio/schema-server v0.0.33 h1:RTeQMIchynAPSQaf61CZBgYHCCpsyDLEAWJn+ZKImIo= github.com/sdcio/schema-server v0.0.33/go.mod h1:q8leN1KhRNTnnqf6yxvkDk5tFl6DAsHcl81usVgYpoI= +github.com/sdcio/sdc-protos v0.0.50 h1:aR6Av1QMTFNXKKxnrz8vlIXQ0y21lxQJEMut1oLd2Bg= +github.com/sdcio/sdc-protos v0.0.50/go.mod h1:huh1QVE023w+reU2Gt6h1f83R3lJidcFLbQje7cMY1M= github.com/sdcio/yang-parser v0.0.12 h1:RSSeqfAOIsJx5Lno5u4/ezyOmQYUduQ22rBfU/mtpJ4= github.com/sdcio/yang-parser v0.0.12/go.mod h1:CBqn3Miq85qmFVGHxHXHLluXkaIOsTzV06IM4DW6+D4= github.com/sirikothe/gotextfsm v1.0.1-0.20200816110946-6aa2cfd355e4 h1:FHUL2HofYJuslFOQdy/JjjP36zxqIpd/dcoiwLMIs7k= diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index d9500575..4530160b 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -46,7 +46,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -111,13 +111,13 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { } conf1 := config1() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } conf2 := config2() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf2, root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -132,7 +132,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/processor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go index 8aaa3fe0..32be93ae 100644 --- a/pkg/tree/processor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -55,7 +55,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsNew) if err != nil { t.Error(err) } @@ -83,12 +83,12 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } @@ -135,17 +135,17 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Error(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, owner1Prio, false, flagsExisting) if err != nil { t.Error(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Error(err) } - testhelper.LoadYgotStructIntoTreeRoot(ctx, &sdcio_schema.Device{Interface: map[string]*sdcio_schema.SdcioModel_Interface{ + loadYgotStructIntoTreeRoot(ctx, &sdcio_schema.Device{Interface: map[string]*sdcio_schema.SdcioModel_Interface{ "ethernet-1/1": { Name: ygot.String("ethernet-1/1"), Description: ygot.String("mydesc"), diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 5b983e9c..0082ab67 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -573,11 +573,11 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 400, false, flagsNew) if err != nil { t.Fatal(err) } @@ -594,7 +594,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 500, false, flagsNew) if err != nil { t.Fatal(err) } diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index 4ea0b7ba..5d1e3e14 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -209,11 +209,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -247,11 +247,11 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config1(), root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, config2(), root, owner2, 10, false, flagsNew) if err != nil { t.Fatal(err) } @@ -345,7 +345,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, d, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -460,7 +460,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { } conf1 := config1() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -475,7 +475,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { running.Patterntest = ygot.String("hallo 0") - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) + _, err = loadYgotStructIntoTreeRoot(ctx, running, root, RunningIntentName, RunningValuesPrio, false, flagsExisting) if err != nil { t.Fatal(err) } @@ -920,7 +920,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { } conf1 := config1() - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -977,7 +977,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - _, err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } From 9cc84744b76a93cbf936efad1f32087e3585987f Mon Sep 17 00:00:00 2001 From: steiler Date: Wed, 11 Feb 2026 12:16:26 +0100 Subject: [PATCH 15/17] alex's comments --- pkg/datastore/datastore_rpc.go | 4 +- pkg/datastore/sync_test.go | 20 +++---- pkg/datastore/target/netconf/nc.go | 4 +- pkg/datastore/transaction_rpc_test.go | 6 +- pkg/datastore/tree_operation_test.go | 6 +- .../tree_operation_validation_test.go | 6 +- pkg/pool/pool.go | 4 +- pkg/pool/virtual_pool.go | 10 ++-- pkg/tree/entry.go | 4 -- pkg/tree/entry_test.go | 54 ++++++++--------- pkg/tree/importer/json/json_tree_importer.go | 13 ++-- .../proto/proto_tree_importer_test.go | 8 +-- pkg/tree/importer/xml/xml_tree_importer.go | 12 ++-- .../importer/xml/xml_tree_importer_test.go | 4 +- pkg/tree/json_test.go | 2 +- pkg/tree/processor_blame_config_test.go | 8 +-- pkg/tree/processor_explicit_delete_test.go | 6 +- pkg/tree/processor_remove_deleted.go | 2 + pkg/tree/processor_reset_flags.go | 20 +++++-- pkg/tree/processor_reset_flags_test.go | 16 ++++- pkg/tree/root_entry.go | 8 +-- pkg/tree/root_entry_test.go | 14 ++--- pkg/tree/sharedEntryAttributes_test.go | 60 +++++++++---------- pkg/tree/tree_context.go | 2 +- pkg/tree/utils.go | 2 +- pkg/tree/validation_entry_leafref_test.go | 4 +- pkg/tree/validation_range_test.go | 12 ++-- pkg/tree/xml_test.go | 4 +- 28 files changed, 165 insertions(+), 150 deletions(-) diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 722e2904..fa8b1814 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -93,7 +93,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca ) scb := schemaClient.NewSchemaClientBound(c.Schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { cancel() @@ -113,7 +113,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca deviationClients: make(map[sdcpb.DataServer_WatchDeviationsServer]string), syncTree: syncTreeRoot, syncTreeMutex: &sync.RWMutex{}, - taskPool: pool.NewSharedTaskPool(ctx, runtime.NumCPU()), + taskPool: pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)), } ds.transactionManager = types.NewTransactionManager(NewDatastoreRollbackAdapter(ds)) diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index 1ba43116..8a1bce3f 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -54,7 +54,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -80,7 +80,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) @@ -160,7 +160,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -186,7 +186,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) @@ -264,7 +264,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -290,7 +290,7 @@ func TestApplyToRunning(t *testing.T) { var v any json.Unmarshal([]byte(confStr), &v) - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(v, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) @@ -373,7 +373,7 @@ func TestApplyToRunning(t *testing.T) { datastore := &Datastore{ syncTreeMutex: &sync.RWMutex{}, syncTree: syncTree, - taskPool: pool.NewSharedTaskPool(ctx, runtime.NumCPU()), + taskPool: pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)), cacheClient: tt.cacheClientFunc(ctrl), sbi: tt.sbiFunc(ctrl), } @@ -390,7 +390,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) resultRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -399,7 +399,7 @@ func TestApplyToRunning(t *testing.T) { d := tt.resultFunc() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = resultRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(d, tree.RunningIntentName, tree.RunningValuesPrio, false), types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatalf("failed to import test config: %v", err) @@ -412,7 +412,7 @@ func TestApplyToRunning(t *testing.T) { fmt.Println(syncTree.String()) - resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetNewFlag().SetUpdateFlag() err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTree.GetRoot(), datastore.taskPool) if err != nil { t.Fatalf("failed to reset flags: %v", err) diff --git a/pkg/datastore/target/netconf/nc.go b/pkg/datastore/target/netconf/nc.go index 84a62439..fea9b7ab 100644 --- a/pkg/datastore/target/netconf/nc.go +++ b/pkg/datastore/target/netconf/nc.go @@ -89,9 +89,9 @@ func (t *ncTarget) GetImportAdapter(ctx context.Context, req *sdcpb.GetDataReque return nil, err } - cmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root(), tree.RunningIntentName, tree.RunningValuesPrio, false) + xmlImport := xml.NewXmlTreeImporter(ncResponse.Doc.Root(), tree.RunningIntentName, tree.RunningValuesPrio, false) - return cmlImport, nil + return xmlImport, nil } func (t *ncTarget) internalGet(ctx context.Context, req *sdcpb.GetDataRequest) (*nctypes.NetconfResponse, error) { diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go index 363698b5..a04bbf2c 100644 --- a/pkg/datastore/transaction_rpc_test.go +++ b/pkg/datastore/transaction_rpc_test.go @@ -140,12 +140,12 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { sbi.EXPECT().Set(gomock.Any(), gomock.Any()).Return(&sdcpb.SetDataResponse{}, nil).AnyTimes() // Setup SyncTree with Running Config - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Fatalf("failed to create sync tree root: %v", err) } - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) // Populate SyncTree with Running config _, err = syncTreeRoot.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(runningAny, tree.RunningIntentName, tree.RunningValuesPrio, false), treetypes.NewUpdateInsertFlags(), vpf) if err != nil { @@ -156,7 +156,7 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { t.Fatalf("failed to finish insertion phase: %v", err) } // Reset flags on syncTree so everything is "existing" - resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(false, true, true) + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetNewFlag().SetUpdateFlag() err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(syncTreeRoot.GetRoot(), vpf) if err != nil { t.Fatalf("failed to reset flags: %v", err) diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index 7f771758..e6c68ae2 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1471,7 +1471,7 @@ func TestDatastore_populateTree(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -1503,7 +1503,7 @@ func TestDatastore_populateTree(t *testing.T) { t.Error(err) } - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := tree.NewOwnerDeleteMarker(tree.NewOwnerDeleteMarkerTaskConfig(tt.intentName, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) if err != nil { @@ -1513,7 +1513,7 @@ func TestDatastore_populateTree(t *testing.T) { newFlag := types.NewUpdateInsertFlags().SetNewFlag() - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, tt.intentReqPath, jsonImporter.NewJsonTreeImporter(jsonConfAny, tt.intentName, tt.intentPrio, tt.nonRevertive), newFlag, sharedPool) diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index aec0b17b..d78e273c 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -187,7 +187,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -198,7 +198,7 @@ func TestDatastore_validateTree(t *testing.T) { importer := json_importer.NewJsonTreeImporter(jsonConf, tt.intentName, tt.intentPrio, tt.nonRevertive) - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, path, importer, flagsNew, vpf) if err != nil { t.Error(err) @@ -209,7 +209,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(ctx, validationConfig, sharedPool) t.Log(root.String()) diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index 18f224eb..7b762dd1 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -34,10 +34,10 @@ type Pool[T any] struct { inflightC *sync.Cond } -// NewWorkerPool creates a new Pool. If workerCount <= 0 it defaults to runtime.NumCPU(). +// NewWorkerPool creates a new Pool. If workerCount <= 0 it defaults to runtime.GOMAXPROCS(0). func NewWorkerPool[T any](parent context.Context, workerCount int) *Pool[T] { if workerCount <= 0 { - workerCount = runtime.NumCPU() + workerCount = runtime.GOMAXPROCS(0) } ctx, cancel := context.WithCancel(parent) p := &Pool[T]{ diff --git a/pkg/pool/virtual_pool.go b/pkg/pool/virtual_pool.go index 0ef4b843..a3f0fdf7 100644 --- a/pkg/pool/virtual_pool.go +++ b/pkg/pool/virtual_pool.go @@ -153,7 +153,7 @@ type VirtualPool struct { // firstErr used for fail-fast firstErr atomic.Pointer[error] // per-virtual inflight counter (matches lifecycle of tasks submitted by this virtual) - inflight int64 + inflight atomic.Int64 // ensure done channel closed only once (for Wait) waitOnce sync.Once // done is closed when the virtual pool is closed for submit and inflight reaches zero @@ -214,7 +214,7 @@ func (vt *virtualTask) Run(ctx context.Context, submit func(Task) error) error { func (v *VirtualPool) Submit(t Task) error { // Increment inflight BEFORE checking closed to avoid race where CloseForSubmit // sees inflight=0 and closes the pool while we are in the middle of submitting. - atomic.AddInt64(&v.inflight, 1) + v.inflight.Add(1) // fast-fail if virtual pool closed for submit if v.closed.Load() { @@ -237,7 +237,7 @@ func (v *VirtualPool) Submit(t Task) error { } func (v *VirtualPool) decrementInflight() { - if remaining := atomic.AddInt64(&v.inflight, -1); remaining == 0 && v.closed.Load() { + if remaining := v.inflight.Add(-1); remaining == 0 && v.closed.Load() { v.waitOnce.Do(func() { close(v.done) }) @@ -258,7 +258,7 @@ func (v *VirtualPool) submitInternal(t Task) error { return ErrVirtualPoolClosed } // increment per-virtual inflight (will be decremented by worker after run) - atomic.AddInt64(&v.inflight, 1) + v.inflight.Add(1) vt := &virtualTask{vp: v, task: t} if err := v.parent.submitWrapped(vt); err != nil { // submission failed: revert inflight @@ -274,7 +274,7 @@ func (v *VirtualPool) submitInternal(t Task) error { func (v *VirtualPool) CloseForSubmit() { v.closed.Store(true) // if nothing inflight, signal Wait() callers that virtual is drained - if atomic.LoadInt64(&v.inflight) == 0 { + if v.inflight.Load() == 0 { v.waitOnce.Do(func() { close(v.done) }) diff --git a/pkg/tree/entry.go b/pkg/tree/entry.go index 080cf504..6dedbe03 100644 --- a/pkg/tree/entry.go +++ b/pkg/tree/entry.go @@ -64,8 +64,6 @@ type Entry interface { // MarkOwnerDelete(o string, onlyIntended bool) // GetDeletes returns the cache-updates that are not updated, have no lower priority value left and hence should be deleted completely GetDeletes(entries []types.DeleteEntry, aggregatePaths bool) ([]types.DeleteEntry, error) - // Walk takes the EntryVisitor and applies it to every Entry in the tree - // Walk(ctx context.Context, v EntryVisitor) error // Validate kicks off validation ValidateLevel(ctx context.Context, resultChan chan<- *types.ValidationResultEntry, stats *types.ValidationStats, vCfg *config.Validation) // validateMandatory the Mandatory schema field @@ -128,8 +126,6 @@ type Entry interface { // ToXML returns the tree and its current state in the XML representation used by netconf ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (*etree.Document, error) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool) (doAdd bool, err error) - // ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree. - // ImportConfig(ctx context.Context, importer importer.ImportConfigAdapter, flags *types.UpdateInsertFlags, poolFactory pool.VirtualPoolFactory) (*types.ImportStats, error) TreeExport(owner string) ([]*tree_persist.TreeElement, error) // DeleteBranch Deletes from the tree, all elements of the PathSlice defined branch of the given owner DeleteBranch(ctx context.Context, path *sdcpb.Path, owner string) (err error) diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index d2e6b0f1..e179648c 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -66,7 +66,7 @@ func Test_Entry(t *testing.T) { u2 := types.NewUpdate(nil, desc, int32(99), "me", int64(444)) u3 := types.NewUpdate(nil, desc, int32(98), "me", int64(88)) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -174,7 +174,7 @@ func Test_Entry_One(t *testing.T) { u3 := types.NewUpdate(nil, desc3, prio50, owner2, ts1) u3_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner2, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -300,7 +300,7 @@ func Test_Entry_Two(t *testing.T) { u1 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) u1_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -495,7 +495,7 @@ func Test_Entry_Three(t *testing.T) { u3r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) u4r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -566,7 +566,7 @@ func Test_Entry_Three(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) @@ -784,7 +784,7 @@ func Test_Entry_Four(t *testing.T) { } // u2o2_1 := types.NewUpdate(p2o2_1, testhelper.GetUIntTvProto(11), prio55, owner2, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -840,7 +840,7 @@ func Test_Entry_Four(t *testing.T) { // indicate that the intent is receiving an update // therefor invalidate all the present entries of the owner / intent - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) @@ -956,7 +956,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -995,7 +995,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Log(root.String()) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1009,7 +1009,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1044,7 +1044,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { } } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1058,7 +1058,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Four", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1099,7 +1099,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { } } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1185,7 +1185,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } u6 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1214,7 +1214,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } } - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) @@ -1480,7 +1480,7 @@ func Test_Schema_Population(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1535,7 +1535,7 @@ func Test_sharedEntryAttributes_SdcpbPath(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1666,7 +1666,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1689,7 +1689,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1705,7 +1705,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1727,7 +1727,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1802,7 +1802,7 @@ func Test_Validation_Deref(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1824,7 +1824,7 @@ func Test_Validation_Deref(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // check if errors are received @@ -1858,7 +1858,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Valid", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1923,7 +1923,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Error(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResult, _ := root.Validate(context.TODO(), validationConfig, sharedPool) // Should have no errors - all keys match their respective patterns @@ -1936,7 +1936,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_Owner", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1980,7 +1980,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_NextHop", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/importer/json/json_tree_importer.go b/pkg/tree/importer/json/json_tree_importer.go index 212c2416..c002251b 100644 --- a/pkg/tree/importer/json/json_tree_importer.go +++ b/pkg/tree/importer/json/json_tree_importer.go @@ -12,7 +12,7 @@ import ( ) type JsonTreeImporter struct { - JsonTreeImporterElement + *JsonTreeImporterElement intentName string priority int32 nonRevertive bool @@ -32,13 +32,10 @@ func (j *JsonTreeImporter) GetName() string { func NewJsonTreeImporter(d any, intentName string, priority int32, nonRevertive bool) *JsonTreeImporter { return &JsonTreeImporter{ - JsonTreeImporterElement: JsonTreeImporterElement{ - data: d, - name: "root", - }, - intentName: intentName, - priority: priority, - nonRevertive: nonRevertive, + JsonTreeImporterElement: newJsonTreeImporterElement("root", d), + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, } } diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 93a882e9..2eb240d5 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -114,7 +114,7 @@ func TestProtoTreeImporter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -130,7 +130,7 @@ func TestProtoTreeImporter(t *testing.T) { jti := jimport.NewJsonTreeImporter(j, "owner1", 5, false) - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, nil, jti, types.NewUpdateInsertFlags(), vpf) if err != nil { t.Fatal(err) @@ -149,7 +149,7 @@ func TestProtoTreeImporter(t *testing.T) { fmt.Println(protoIntent.PrettyString(" ")) - tcNew := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tcNew := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) rootNew, err := tree.NewTreeRoot(ctx, tcNew) if err != nil { t.Error(err) @@ -157,7 +157,7 @@ func TestProtoTreeImporter(t *testing.T) { protoAdapter := NewProtoTreeImporter(protoIntent) - vpf2 := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf2 := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = rootNew.ImportConfig(ctx, nil, protoAdapter, types.NewUpdateInsertFlags(), vpf2) if err != nil { t.Error(err) diff --git a/pkg/tree/importer/xml/xml_tree_importer.go b/pkg/tree/importer/xml/xml_tree_importer.go index 4973d4b9..f2c5d0ad 100644 --- a/pkg/tree/importer/xml/xml_tree_importer.go +++ b/pkg/tree/importer/xml/xml_tree_importer.go @@ -9,7 +9,7 @@ import ( ) type XmlTreeImporter struct { - XmlTreeImporterElement + *XmlTreeImporterElement intentName string priority int32 nonRevertive bool @@ -17,12 +17,10 @@ type XmlTreeImporter struct { func NewXmlTreeImporter(d *etree.Element, intentName string, priority int32, nonRevertive bool) *XmlTreeImporter { return &XmlTreeImporter{ - XmlTreeImporterElement: XmlTreeImporterElement{ - elem: d, - }, - intentName: intentName, - priority: priority, - nonRevertive: nonRevertive, + XmlTreeImporterElement: NewXmlTreeImporterElement(d), + intentName: intentName, + priority: priority, + nonRevertive: nonRevertive, } } diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index 227aa820..65c99a41 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -79,7 +79,7 @@ func TestXmlTreeImporter(t *testing.T) { } ctx := context.Background() - tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -94,7 +94,7 @@ func TestXmlTreeImporter(t *testing.T) { t.Fatal(err) } - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, nil, NewXmlTreeImporter(&inputDoc.Element, "owner1", 5, false), types.NewUpdateInsertFlags(), sharedPool) sharedPool.CloseForSubmit() diff --git a/pkg/tree/json_test.go b/pkg/tree/json_test.go index fd695677..e9ad98a7 100644 --- a/pkg/tree/json_test.go +++ b/pkg/tree/json_test.go @@ -380,7 +380,7 @@ func TestToJsonTable(t *testing.T) { ctx := context.Background() - tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index b5a9646c..beded2cd 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -39,7 +39,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -104,7 +104,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -152,7 +152,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { treeRoot := tt.r(t) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) vPool := sharedPool.NewVirtualPool(pool.VirtualFailFast) bp := NewBlameConfigProcessor(NewBlameConfigProcessorConfig(tt.includeDefaults)) diff --git a/pkg/tree/processor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go index 32be93ae..ff7e44df 100644 --- a/pkg/tree/processor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -48,7 +48,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -76,7 +76,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -128,7 +128,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/processor_remove_deleted.go b/pkg/tree/processor_remove_deleted.go index d029948c..0827b31c 100644 --- a/pkg/tree/processor_remove_deleted.go +++ b/pkg/tree/processor_remove_deleted.go @@ -42,6 +42,8 @@ func (r *RemoveDeletedProcessorParameters) GetDeleteStatsCount() int64 { // GetZeroLengthLeafVariantEntries returns the entries that have zero-length leaf variant entries after removal func (r *RemoveDeletedProcessorParameters) GetZeroLengthLeafVariantEntries() []Entry { + r.zeroLeafEntryElementsLock.Lock() + defer r.zeroLeafEntryElementsLock.Unlock() return r.zeroLeafEntryElements } diff --git a/pkg/tree/processor_reset_flags.go b/pkg/tree/processor_reset_flags.go index b1b462b9..623fb410 100644 --- a/pkg/tree/processor_reset_flags.go +++ b/pkg/tree/processor_reset_flags.go @@ -26,15 +26,27 @@ type ResetFlagsProcessorParameters struct { adjustedFlagsCount atomic.Int64 } -func NewResetFlagsProcessorParameters(deleteFlag, newFlag, updateFlag bool) *ResetFlagsProcessorParameters { +func NewResetFlagsProcessorParameters() *ResetFlagsProcessorParameters { return &ResetFlagsProcessorParameters{ - deleteFlag: deleteFlag, - newFlag: newFlag, - updateFlag: updateFlag, adjustedFlagsCount: atomic.Int64{}, } } +func (r *ResetFlagsProcessorParameters) SetDeleteFlag() *ResetFlagsProcessorParameters { + r.deleteFlag = true + return r +} + +func (r *ResetFlagsProcessorParameters) SetNewFlag() *ResetFlagsProcessorParameters { + r.newFlag = true + return r +} + +func (r *ResetFlagsProcessorParameters) SetUpdateFlag() *ResetFlagsProcessorParameters { + r.updateFlag = true + return r +} + // GetAdjustedFlagsCount returns the number of flags that were adjusted func (r *ResetFlagsProcessorParameters) GetAdjustedFlagsCount() int64 { return r.adjustedFlagsCount.Load() diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go index 642de8dd..8b6156a8 100644 --- a/pkg/tree/processor_reset_flags_test.go +++ b/pkg/tree/processor_reset_flags_test.go @@ -42,7 +42,7 @@ func TestResetFlagsProcessorRun(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -89,7 +89,17 @@ func TestResetFlagsProcessorRun(t *testing.T) { ctx := context.Background() // Create processor parameters - params := NewResetFlagsProcessorParameters(tt.deleteFlag, tt.newFlag, tt.updateFlag) + params := NewResetFlagsProcessorParameters() + + if tt.deleteFlag { + params.SetDeleteFlag() + } + if tt.newFlag { + params.SetNewFlag() + } + if tt.updateFlag { + params.SetUpdateFlag() + } // Create processor processor := NewResetFlagsProcessor(params) @@ -100,7 +110,7 @@ func TestResetFlagsProcessorRun(t *testing.T) { // This is a simplified version - you may need to adjust based on your actual Entry implementation // Create a virtual pool for testing - taskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + taskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) root := tt.tree() diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index 80582fa0..eaece2bd 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -52,7 +52,7 @@ func (r *RootEntry) stringToDisk(filename string) error { } func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { - tc := r.treeContext.deepCopy() + tc := r.GetTreeContext().deepCopy() se, err := r.sharedEntryAttributes.deepCopy(tc, nil) if err != nil { return nil, err @@ -183,7 +183,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32) (*tree_persist.Inte return nil, err } - explicitDeletes := r.treeContext.explicitDeletes.GetByIntentName(owner).ToPathSlice() + explicitDeletes := r.GetTreeContext().explicitDeletes.GetByIntentName(owner).ToPathSlice() var rootExportEntry *tree_persist.TreeElement if len(treeExport) != 0 { @@ -195,7 +195,7 @@ func (r *RootEntry) TreeExport(owner string, priority int32) (*tree_persist.Inte IntentName: owner, Root: rootExportEntry, Priority: priority, - NonRevertive: r.treeContext.nonRevertiveInfo[owner], + NonRevertive: r.GetTreeContext().nonRevertiveInfo[owner], ExplicitDeletes: explicitDeletes, }, nil } @@ -239,7 +239,7 @@ func (r *RootEntry) FinishInsertionPhase(ctx context.Context) error { edpsc := ExplicitDeleteProcessorStatCollection{} // apply the explicit deletes - for deletePathPrio := range r.treeContext.explicitDeletes.Items() { + for deletePathPrio := range r.GetTreeContext().explicitDeletes.Items() { params := NewExplicitDeleteTaskParameters(deletePathPrio.GetOwner(), deletePathPrio.GetPrio()) diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 0082ab67..3d56c566 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -26,7 +26,7 @@ import ( func TestRootEntry_TreeExport(t *testing.T) { owner1 := "owner1" owner2 := "owner2" - tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.NumCPU())) + tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) type args struct { owner string @@ -397,7 +397,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -434,7 +434,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) type fields struct { sharedEntryAttributes func(t *testing.T) *sharedEntryAttributes @@ -521,7 +521,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ImportConfigProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonAny, "owner1", 5, false), types.NewUpdateInsertFlags()) err = ImportConfigProcessor.Run(ctx, s, vpf) if err != nil { @@ -568,7 +568,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { { name: "One", rootEntry: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -589,7 +589,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { }, owner: owner1, want: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -611,7 +611,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { root := tt.rootEntry(t) got := root.GetUpdatesForOwner(tt.owner).ToPathAndUpdateSlice() - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) resultRoot, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index 36f1aac5..f55f1040 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -37,7 +37,7 @@ func Test_sharedEntryAttributes_checkAndCreateKeysAsLeafs(t *testing.T) { ctx := context.Background() scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -93,7 +93,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { { name: "just rootEntry", root: func() *RootEntry { - tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.NumCPU())) + tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) r := &RootEntry{ sharedEntryAttributes: &sharedEntryAttributes{ @@ -123,7 +123,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -146,7 +146,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Error(err) @@ -204,7 +204,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -242,7 +242,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -339,7 +339,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -453,7 +453,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -632,7 +632,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -655,7 +655,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) @@ -672,7 +672,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MustStatement = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) @@ -755,7 +755,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -778,7 +778,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) @@ -794,7 +794,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { valConfig := validationConfig.DeepCopy() valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MustStatement = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) @@ -867,7 +867,7 @@ func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -913,7 +913,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -945,7 +945,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1006,7 +1006,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1020,7 +1020,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { }, }, } - err = testhelper.LoadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, flagsNew) + _, err = loadYgotStructIntoTreeRoot(ctx, conf1, root, owner1, 5, false, flagsNew) if err != nil { t.Fatal(err) } @@ -1044,7 +1044,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { validationConfig.DisabledValidators.DisableAll() validationConfig.DisabledValidators.Mandatory = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) validationResults, _ := root.Validate(ctx, validationConfig, sharedPool) @@ -1125,7 +1125,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1166,13 +1166,13 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println("\nTreeExport:") fmt.Println(string(persistByte)) - tcNew := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tcNew := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) newRoot, err := NewTreeRoot(ctx, tcNew) if err != nil { t.Fatal(err) } - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = newRoot.ImportConfig(ctx, &sdcpb.Path{}, proto.NewProtoTreeImporter(treepersist), flagsExisting, vpf) if err != nil { t.Error(err) @@ -1180,7 +1180,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { } // mark owner delete - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner1, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) @@ -1301,7 +1301,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1324,7 +1324,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) @@ -1341,7 +1341,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MaxElements = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) t.Log(strings.Join(result.ErrorsStr(), "\n")) @@ -1471,7 +1471,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1494,7 +1494,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) @@ -1511,7 +1511,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { valConfig.DisabledValidators.DisableAll() valConfig.DisabledValidators.MaxElements = false - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) result, _ := root.Validate(ctx, valConfig, sharedPool) t.Log(strings.Join(result.ErrorsStr(), "\n")) diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index 9da8ab59..b01bdd09 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -35,7 +35,7 @@ func (t *TreeContext) deepCopy() *TreeContext { } // deepcopy nonRevertiveInfo - m := map[string]bool{} + m := make(map[string]bool, len(t.nonRevertiveInfo)) for k, v := range t.nonRevertiveInfo { m[k] = v } diff --git a/pkg/tree/utils.go b/pkg/tree/utils.go index ce938649..06edda8c 100644 --- a/pkg/tree/utils.go +++ b/pkg/tree/utils.go @@ -33,7 +33,7 @@ func loadYgotStructIntoTreeRoot(ctx context.Context, gs ygot.GoStruct, root *Roo return nil, err } - stp := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + stp := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) importProcessor := NewImportConfigProcessor(jsonImporter.NewJsonTreeImporter(jsonConfAny, owner, prio, nonRevertive), flags) err = importProcessor.Run(ctx, root.sharedEntryAttributes, stp) diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 5c8fc947..7ffc0005 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -215,7 +215,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -238,7 +238,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { newFlag := types.NewUpdateInsertFlags() - vpf := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jsonImporter.NewJsonTreeImporter(jsonConfAny, owner1, 500, false), newFlag, vpf) if err != nil { t.Fatal(err) diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 8716069a..1794be47 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -27,7 +27,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) @@ -67,7 +67,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) if err != nil { t.Error(err) @@ -157,7 +157,7 @@ func TestValidate_RangesSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -181,7 +181,7 @@ func TestValidate_RangesSigned(t *testing.T) { // new json tree importer jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) // import via importer _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) @@ -293,7 +293,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -317,7 +317,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { // new json tree importer jimporter := json_importer.NewJsonTreeImporter(jsonConfig, "owner1", 5, false) - sharedPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) // import via importer _, err = root.ImportConfig(ctx, &sdcpb.Path{}, jimporter, types.NewUpdateInsertFlags(), sharedPool) diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index f82deccc..22a06618 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -574,7 +574,7 @@ func TestToXMLTable(t *testing.T) { converter := utils.NewConverter(scb) - tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.NumCPU())) + tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -593,7 +593,7 @@ func TestToXMLTable(t *testing.T) { fmt.Println(root.String()) if tt.newConfig != nil { - sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.NumCPU()) + sharedTaskPool := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) ownerDeleteMarker := NewOwnerDeleteMarker(NewOwnerDeleteMarkerTaskConfig(owner, false)) err = ownerDeleteMarker.Run(root.GetRoot(), sharedTaskPool) From 6fb61b899f32b3a91b6edc45291a904ea2be4ee4 Mon Sep 17 00:00:00 2001 From: steiler Date: Wed, 11 Feb 2026 15:59:25 +0100 Subject: [PATCH 16/17] cleanup TreeContext --- pkg/datastore/datastore_rpc.go | 2 +- pkg/datastore/deviations.go | 10 ------ pkg/datastore/intent_rpc.go | 2 +- pkg/datastore/sync.go | 4 +-- pkg/datastore/sync_test.go | 8 ++--- pkg/datastore/transaction_rpc.go | 5 +-- pkg/datastore/transaction_rpc_test.go | 2 +- pkg/datastore/tree_operation_test.go | 2 +- .../tree_operation_validation_test.go | 2 +- pkg/tree/entry_test.go | 34 +++++++++--------- .../proto/proto_tree_importer_test.go | 4 +-- .../importer/xml/xml_tree_importer_test.go | 2 +- pkg/tree/json_test.go | 2 +- pkg/tree/processor_blame_config_test.go | 6 ++-- pkg/tree/processor_explicit_delete_test.go | 6 ++-- pkg/tree/processor_importer.go | 3 -- pkg/tree/processor_reset_flags_test.go | 2 +- pkg/tree/root_entry.go | 9 ----- pkg/tree/root_entry_test.go | 12 +++---- pkg/tree/sharedEntryAttributes_test.go | 35 +++++++++---------- pkg/tree/tree_context.go | 23 +----------- pkg/tree/validation_entry_leafref_test.go | 2 +- pkg/tree/validation_range_test.go | 6 ++-- pkg/tree/xml_test.go | 26 +------------- 24 files changed, 69 insertions(+), 140 deletions(-) diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index fa8b1814..f1efe758 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -93,7 +93,7 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca ) scb := schemaClient.NewSchemaClientBound(c.Schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { cancel() diff --git a/pkg/datastore/deviations.go b/pkg/datastore/deviations.go index b3bbb1ee..54f56867 100644 --- a/pkg/datastore/deviations.go +++ b/pkg/datastore/deviations.go @@ -123,16 +123,6 @@ func (d *Datastore) SendDeviations(ctx context.Context, ch <-chan *treetypes.Dev continue } - if log := log.V(logger.VTrace); log.Enabled() { - if deviation.Reason() == treetypes.DeviationReasonNotApplied { // TODO add check for trace level Trace - current := "nil" - if deviation.CurrentValue() != nil { - current = deviation.CurrentValue().ToString() - } - log.Info("NOT APPLIED", "path", deviation.Path().ToXPath(false), "actual value", current, "expected value", deviation.ExpectedValue().ToString(), "intent", deviation.IntentName()) - } - } - err := dc.Send(&sdcpb.WatchDeviationResponse{ Name: d.config.Name, Intent: deviation.IntentName(), diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index bb112e21..73cbed4e 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -66,7 +66,7 @@ func (d *Datastore) GetIntent(ctx context.Context, intentName string) (GetIntent } // otherwise consult cache - root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, intentName, d.taskPool)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(d.schemaClient, d.taskPool)) if err != nil { return nil, err } diff --git a/pkg/datastore/sync.go b/pkg/datastore/sync.go index 57c243d5..2c85cda2 100644 --- a/pkg/datastore/sync.go +++ b/pkg/datastore/sync.go @@ -72,7 +72,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } // run reset flags processor to reset flags - resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters(true, true, true) + resetFlagsProcessorParams := tree.NewResetFlagsProcessorParameters().SetDeleteFlag().SetNewFlag().SetUpdateFlag() err = tree.NewResetFlagsProcessor(resetFlagsProcessorParams).Run(d.syncTree.GetRoot(), d.taskPool) if err != nil { return err @@ -99,7 +99,7 @@ func (d *Datastore) ApplyToRunning(ctx context.Context, deletes []*sdcpb.Path, i } func (d *Datastore) NewEmptyTree(ctx context.Context) (*tree.RootEntry, error) { - tc := tree.NewTreeContext(d.schemaClient, tree.RunningIntentName, d.taskPool) + tc := tree.NewTreeContext(d.schemaClient, d.taskPool) newTree, err := tree.NewTreeRoot(ctx, tc) if err != nil { return nil, err diff --git a/pkg/datastore/sync_test.go b/pkg/datastore/sync_test.go index 8a1bce3f..f6faf4d7 100644 --- a/pkg/datastore/sync_test.go +++ b/pkg/datastore/sync_test.go @@ -54,7 +54,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -160,7 +160,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -264,7 +264,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { @@ -390,7 +390,7 @@ func TestApplyToRunning(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) resultRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 577425a7..9e5c7fd7 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -72,7 +72,7 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa ctx = logf.IntoContext(ctx, log) // create a new TreeContext - tc := tree.NewTreeContext(d.schemaClient, d.Name(), d.taskPool) + tc := tree.NewTreeContext(d.schemaClient, d.taskPool) // create a new TreeRoot to collect validate and hand to SBI.Set() root, err := tree.NewTreeRoot(ctx, tc) @@ -80,9 +80,6 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa return nil, err } - // set TreeContext actual owner to the const of ReplaceIntentName - tc.SetActualOwner(tree.ReplaceIntentName) - // store the actual / old running in the transaction runningProto, err := d.cacheClient.IntentGet(ctx, tree.RunningIntentName) if err != nil { diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go index a04bbf2c..2dc0ee12 100644 --- a/pkg/datastore/transaction_rpc_test.go +++ b/pkg/datastore/transaction_rpc_test.go @@ -140,7 +140,7 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { sbi.EXPECT().Set(gomock.Any(), gomock.Any()).Return(&sdcpb.SetDataResponse{}, nil).AnyTimes() // Setup SyncTree with Running Config - tc := tree.NewTreeContext(scb, tree.RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) syncTreeRoot, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Fatalf("failed to create sync tree root: %v", err) diff --git a/pkg/datastore/tree_operation_test.go b/pkg/datastore/tree_operation_test.go index e6c68ae2..a3e3be45 100644 --- a/pkg/datastore/tree_operation_test.go +++ b/pkg/datastore/tree_operation_test.go @@ -1471,7 +1471,7 @@ func TestDatastore_populateTree(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/datastore/tree_operation_validation_test.go b/pkg/datastore/tree_operation_validation_test.go index d78e273c..88acd4ef 100644 --- a/pkg/datastore/tree_operation_validation_test.go +++ b/pkg/datastore/tree_operation_validation_test.go @@ -187,7 +187,7 @@ func TestDatastore_validateTree(t *testing.T) { t.Error(err) } - tc := tree.NewTreeContext(scb, tt.intentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) diff --git a/pkg/tree/entry_test.go b/pkg/tree/entry_test.go index e179648c..c497b2d6 100644 --- a/pkg/tree/entry_test.go +++ b/pkg/tree/entry_test.go @@ -66,7 +66,7 @@ func Test_Entry(t *testing.T) { u2 := types.NewUpdate(nil, desc, int32(99), "me", int64(444)) u3 := types.NewUpdate(nil, desc, int32(98), "me", int64(88)) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -174,7 +174,7 @@ func Test_Entry_One(t *testing.T) { u3 := types.NewUpdate(nil, desc3, prio50, owner2, ts1) u3_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner2, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -300,7 +300,7 @@ func Test_Entry_Two(t *testing.T) { u1 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) u1_1 := types.NewUpdate(nil, testhelper.GetUIntTvProto(10), prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -495,7 +495,7 @@ func Test_Entry_Three(t *testing.T) { u3r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) u4r := types.NewUpdate(nil, desc3, RunningValuesPrio, RunningIntentName, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -784,7 +784,7 @@ func Test_Entry_Four(t *testing.T) { } // u2o2_1 := types.NewUpdate(p2o2_1, testhelper.GetUIntTvProto(11), prio55, owner2, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -956,7 +956,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1009,7 +1009,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1058,7 +1058,7 @@ func Test_Validation_Leaflist_Min_Max(t *testing.T) { t.Run("Test Leaflist min- & max- elements - Four", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1185,7 +1185,7 @@ func Test_Entry_Delete_Aggregation(t *testing.T) { } u6 := types.NewUpdate(nil, desc3, prio50, owner1, ts1) - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1480,7 +1480,7 @@ func Test_Schema_Population(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1535,7 +1535,7 @@ func Test_sharedEntryAttributes_SdcpbPath(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, "foo", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1666,7 +1666,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1705,7 +1705,7 @@ func Test_Validation_String_Pattern(t *testing.T) { t.Run("Test_Validation_String_Pattern - Two", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1802,7 +1802,7 @@ func Test_Validation_Deref(t *testing.T) { t.Run("Test_Validation_String_Pattern - One", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1858,7 +1858,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Valid", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1936,7 +1936,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_Owner", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1980,7 +1980,7 @@ func Test_Validation_MultiKey_Pattern(t *testing.T) { t.Run("MultiKey_Pattern_Invalid_NextHop", func(t *testing.T) { - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/importer/proto/proto_tree_importer_test.go b/pkg/tree/importer/proto/proto_tree_importer_test.go index 2eb240d5..910f3797 100644 --- a/pkg/tree/importer/proto/proto_tree_importer_test.go +++ b/pkg/tree/importer/proto/proto_tree_importer_test.go @@ -114,7 +114,7 @@ func TestProtoTreeImporter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := tree.NewTreeRoot(ctx, tc) if err != nil { t.Error(err) @@ -149,7 +149,7 @@ func TestProtoTreeImporter(t *testing.T) { fmt.Println(protoIntent.PrettyString(" ")) - tcNew := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tcNew := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) rootNew, err := tree.NewTreeRoot(ctx, tcNew) if err != nil { t.Error(err) diff --git a/pkg/tree/importer/xml/xml_tree_importer_test.go b/pkg/tree/importer/xml/xml_tree_importer_test.go index 65c99a41..10cf987f 100644 --- a/pkg/tree/importer/xml/xml_tree_importer_test.go +++ b/pkg/tree/importer/xml/xml_tree_importer_test.go @@ -79,7 +79,7 @@ func TestXmlTreeImporter(t *testing.T) { } ctx := context.Background() - tc := tree.NewTreeContext(scb, "test", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := tree.NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/tree/json_test.go b/pkg/tree/json_test.go index e9ad98a7..007ccab6 100644 --- a/pkg/tree/json_test.go +++ b/pkg/tree/json_test.go @@ -380,7 +380,7 @@ func TestToJsonTable(t *testing.T) { ctx := context.Background() - tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/processor_blame_config_test.go b/pkg/tree/processor_blame_config_test.go index beded2cd..75f2aee2 100644 --- a/pkg/tree/processor_blame_config_test.go +++ b/pkg/tree/processor_blame_config_test.go @@ -39,7 +39,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -104,7 +104,7 @@ func Test_sharedEntryAttributes_BlameConfig(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/processor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go index ff7e44df..eaffe79a 100644 --- a/pkg/tree/processor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -48,7 +48,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -76,7 +76,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -128,7 +128,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/processor_importer.go b/pkg/tree/processor_importer.go index 9b459976..1e1543e3 100644 --- a/pkg/tree/processor_importer.go +++ b/pkg/tree/processor_importer.go @@ -66,9 +66,6 @@ func (p *ImportConfigProcessor) GetStats() *types.ImportStats { } func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory pool.VirtualPoolFactory) error { - // set actual owner - e.GetTreeContext().SetActualOwner(p.importer.GetName()) - // store non revertive info e.GetTreeContext().nonRevertiveInfo[p.importer.GetName()] = p.importer.GetNonRevertive() diff --git a/pkg/tree/processor_reset_flags_test.go b/pkg/tree/processor_reset_flags_test.go index 8b6156a8..424f4b17 100644 --- a/pkg/tree/processor_reset_flags_test.go +++ b/pkg/tree/processor_reset_flags_test.go @@ -42,7 +42,7 @@ func TestResetFlagsProcessorRun(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, RunningIntentName, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/root_entry.go b/pkg/tree/root_entry.go index eaece2bd..02e937c2 100644 --- a/pkg/tree/root_entry.go +++ b/pkg/tree/root_entry.go @@ -37,11 +37,6 @@ func NewTreeRoot(ctx context.Context, tc *TreeContext) (*RootEntry, error) { sharedEntryAttributes: sea, } - err = tc.SetRoot(sea) - if err != nil { - return nil, err - } - return root, nil } @@ -62,10 +57,6 @@ func (r *RootEntry) DeepCopy(ctx context.Context) (*RootEntry, error) { sharedEntryAttributes: se, } - err = tc.SetRoot(result.sharedEntryAttributes) - if err != nil { - return nil, err - } return result, nil } diff --git a/pkg/tree/root_entry_test.go b/pkg/tree/root_entry_test.go index 3d56c566..6303e2bd 100644 --- a/pkg/tree/root_entry_test.go +++ b/pkg/tree/root_entry_test.go @@ -26,7 +26,7 @@ import ( func TestRootEntry_TreeExport(t *testing.T) { owner1 := "owner1" owner2 := "owner2" - tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) + tc := NewTreeContext(nil, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) type args struct { owner string @@ -397,7 +397,7 @@ func TestRootEntry_DeleteSubtreePaths(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -434,7 +434,7 @@ func TestRootEntry_AddUpdatesRecursive(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) type fields struct { sharedEntryAttributes func(t *testing.T) *sharedEntryAttributes @@ -568,7 +568,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { { name: "One", rootEntry: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -589,7 +589,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { }, owner: owner1, want: func(t *testing.T) *RootEntry { - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -611,7 +611,7 @@ func TestRootEntry_GetUpdatesForOwner(t *testing.T) { root := tt.rootEntry(t) got := root.GetUpdatesForOwner(tt.owner).ToPathAndUpdateSlice() - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) resultRoot, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) diff --git a/pkg/tree/sharedEntryAttributes_test.go b/pkg/tree/sharedEntryAttributes_test.go index f55f1040..c569cd55 100644 --- a/pkg/tree/sharedEntryAttributes_test.go +++ b/pkg/tree/sharedEntryAttributes_test.go @@ -37,7 +37,7 @@ func Test_sharedEntryAttributes_checkAndCreateKeysAsLeafs(t *testing.T) { ctx := context.Background() scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, "intent1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -93,7 +93,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { { name: "just rootEntry", root: func() *RootEntry { - tc := NewTreeContext(nil, owner1, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) + tc := NewTreeContext(nil, pool.NewSharedTaskPool(context.Background(), runtime.GOMAXPROCS(0))) r := &RootEntry{ sharedEntryAttributes: &sharedEntryAttributes{ @@ -123,7 +123,7 @@ func Test_sharedEntryAttributes_DeepCopy(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -204,7 +204,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -242,7 +242,7 @@ func Test_sharedEntryAttributes_DeleteSubtree(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -339,7 +339,7 @@ func Test_sharedEntryAttributes_GetListChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -453,7 +453,7 @@ func Test_sharedEntryAttributes_GetDeviations(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -632,7 +632,7 @@ func Test_sharedEntryAttributes_MustCount(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -755,7 +755,7 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -809,7 +809,6 @@ func Test_sharedEntryAttributes_MustCountDoubleKey(t *testing.T) { func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { ctx := context.TODO() - owner1 := "owner1" tests := []struct { name string @@ -867,7 +866,7 @@ func Test_sharedEntryAttributes_getOrCreateChilds(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -913,7 +912,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -945,7 +944,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1006,7 +1005,7 @@ func Test_sharedEntryAttributes_validateMandatory(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1125,7 +1124,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { t.Fatal(err) } - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) @@ -1166,7 +1165,7 @@ func Test_sharedEntryAttributes_ReApply(t *testing.T) { fmt.Println("\nTreeExport:") fmt.Println(string(persistByte)) - tcNew := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tcNew := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) newRoot, err := NewTreeRoot(ctx, tcNew) if err != nil { t.Fatal(err) @@ -1301,7 +1300,7 @@ func Test_sharedEntryAttributes_validateMinMaxElements(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { @@ -1471,7 +1470,7 @@ func Test_sharedEntryAttributes_validateMinMaxElementsDoubleKey(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/tree_context.go b/pkg/tree/tree_context.go index b01bdd09..e0297de5 100644 --- a/pkg/tree/tree_context.go +++ b/pkg/tree/tree_context.go @@ -1,26 +1,21 @@ package tree import ( - "fmt" - schemaClient "github.com/sdcio/data-server/pkg/datastore/clients/schema" "github.com/sdcio/data-server/pkg/pool" sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) type TreeContext struct { - root Entry // the trees root element schemaClient schemaClient.SchemaClientBound - actualOwner string nonRevertiveInfo map[string]bool explicitDeletes *DeletePathSet poolFactory pool.VirtualPoolFactory } -func NewTreeContext(sc schemaClient.SchemaClientBound, actualOwner string, poolFactory pool.VirtualPoolFactory) *TreeContext { +func NewTreeContext(sc schemaClient.SchemaClientBound, poolFactory pool.VirtualPoolFactory) *TreeContext { return &TreeContext{ schemaClient: sc, - actualOwner: actualOwner, nonRevertiveInfo: map[string]bool{}, explicitDeletes: NewDeletePaths(), poolFactory: poolFactory, @@ -64,19 +59,3 @@ func (t *TreeContext) AddNonRevertiveInfo(intent string, nonRevertive bool) { func (t *TreeContext) IsNonRevertiveIntent(intent string) bool { return t.nonRevertiveInfo[intent] } - -func (t *TreeContext) SetRoot(e Entry) error { - if t.root != nil { - return fmt.Errorf("trying to set treecontexts root, although it is already set") - } - t.root = e - return nil -} - -func (t *TreeContext) GetActualOwner() string { - return t.actualOwner -} - -func (t *TreeContext) SetActualOwner(owner string) { - t.actualOwner = owner -} diff --git a/pkg/tree/validation_entry_leafref_test.go b/pkg/tree/validation_entry_leafref_test.go index 7ffc0005..6ed11249 100644 --- a/pkg/tree/validation_entry_leafref_test.go +++ b/pkg/tree/validation_entry_leafref_test.go @@ -215,7 +215,7 @@ func Test_sharedEntryAttributes_validateLeafRefs(t *testing.T) { t.Fatal(err) } scb := schemaClient.NewSchemaClientBound(schema, sc) - tc := NewTreeContext(scb, owner1, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { diff --git a/pkg/tree/validation_range_test.go b/pkg/tree/validation_range_test.go index 1794be47..7b2d39c7 100644 --- a/pkg/tree/validation_range_test.go +++ b/pkg/tree/validation_range_test.go @@ -27,7 +27,7 @@ func TestValidate_Range_SDC_Schema(t *testing.T) { t.Error(err) } - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) @@ -157,7 +157,7 @@ func TestValidate_RangesSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) @@ -293,7 +293,7 @@ func TestValidate_RangesUnSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // the tree context - tc := NewTreeContext(scb, "owner1", pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) // the tree root root, err := NewTreeRoot(ctx, tc) diff --git a/pkg/tree/xml_test.go b/pkg/tree/xml_test.go index 22a06618..8084a4ff 100644 --- a/pkg/tree/xml_test.go +++ b/pkg/tree/xml_test.go @@ -546,35 +546,11 @@ func TestToXMLTable(t *testing.T) { } owner := "owner1" - // var runningCacheUpds []*types.Update - // if tt.runningConfig != nil { - // runningSdcpbUpds, err := tt.runningConfig(context.Background(), utils.NewConverter(scb)) - // if err != nil { - // t.Error(err) - // } - // runningCacheUpds, err = utils.SdcpbUpdatesToCacheUpdates(runningSdcpbUpds, RunningIntentName, RunningValuesPrio) - // if err != nil { - // t.Error(err) - // } - // } - - // var intendedCacheUpds []*types.Update - // if tt.existingConfig != nil { - // intendedSdcpbUpds, err := tt.existingConfig(context.Background(), utils.NewConverter(scb)) - // if err != nil { - // t.Error(err) - // } - // intendedCacheUpds, err = utils.SdcpbUpdatesToCacheUpdates(intendedSdcpbUpds, owner, 5) - // if err != nil { - // t.Error(err) - // } - // } - ctx := context.Background() converter := utils.NewConverter(scb) - tc := NewTreeContext(scb, owner, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) + tc := NewTreeContext(scb, pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0))) root, err := NewTreeRoot(ctx, tc) if err != nil { t.Fatal(err) From 08e2191333ac72627664030f914a6fc7941e38e5 Mon Sep 17 00:00:00 2001 From: steiler Date: Wed, 11 Feb 2026 19:51:50 +0100 Subject: [PATCH 17/17] remove the complicated locking forimporting leaflists --- pkg/tree/processor_explicit_delete_test.go | 2 +- pkg/tree/processor_importer.go | 58 ++++++++++------------ 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/pkg/tree/processor_explicit_delete_test.go b/pkg/tree/processor_explicit_delete_test.go index eaffe79a..2b3939a8 100644 --- a/pkg/tree/processor_explicit_delete_test.go +++ b/pkg/tree/processor_explicit_delete_test.go @@ -278,7 +278,7 @@ func TestExplicitDeleteVisitor_Visit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root := tt.root() - root.treeContext.AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) + root.GetTreeContext().AddExplicitDeletes(owner2, tt.priority, tt.explicitDeletes) err := root.FinishInsertionPhase(ctx) if err != nil { diff --git a/pkg/tree/processor_importer.go b/pkg/tree/processor_importer.go index 1e1543e3..e2cc8ac1 100644 --- a/pkg/tree/processor_importer.go +++ b/pkg/tree/processor_importer.go @@ -21,12 +21,12 @@ type importConfigTask struct { } type ImportConfigProcessorParams struct { - intentName string - intentPrio int32 - insertFlags *types.UpdateInsertFlags - treeContext *TreeContext - leafListLock *sync.Map - stats *types.ImportStats + intentName string + intentPrio int32 + insertFlags *types.UpdateInsertFlags + treeContext *TreeContext + leafListTracker *sync.Map + stats *types.ImportStats } func NewParameters( @@ -38,12 +38,12 @@ func NewParameters( stats *types.ImportStats, ) *ImportConfigProcessorParams { return &ImportConfigProcessorParams{ - intentName: intentName, - intentPrio: intentPrio, - insertFlags: insertFlags, - treeContext: treeContext, - leafListLock: leafListLock, - stats: stats, + intentName: intentName, + intentPrio: intentPrio, + insertFlags: insertFlags, + treeContext: treeContext, + leafListTracker: leafListLock, + stats: stats, } } @@ -90,9 +90,6 @@ func (p *ImportConfigProcessor) Run(ctx context.Context, e Entry, poolFactory po if err := workerPool.FirstError(); err != nil { return err } - // TODO: support tolerant mode? Processor usually decides what to return based on pool mode, - // but FirstError() works for fail-fast. For tolerant, we might want Errors(). - // But ImportConfig usually stops on error? return nil } @@ -180,34 +177,29 @@ func (task importConfigTask) Run(ctx context.Context, submit func(pool.Task) err return nil case *sdcpb.SchemaElem_Leaflist: - // for the leaflist, since in XML the leaf list elements are independet elements, we need to make - // sure that the first element is basically resetting the leaflist and all consecutive elemts are then - // added to the already resettet leaflist. - // strategy here is to create a mutex lock it and try to store it in the leafListLock map. - // if the mutex was then stored, we're the first goroutine and need to reset. If we get a different mutex back - // and the the loaded var is set to true, we should not reset the list and trxy to lock the returned mutex. + // For leaf lists we need to make sure the first insertion resets the leaf list, all consecutive insertions do add to it. + // To do so, we have the leafListTracker map that indicates if a leaf list was already reset and the first insertion was done or not. + // The key for the map is the combination of the parent entry and the leaf list name, so we can have multiple leaf lists under the same parent without + // interference. - // create a mutex and lock it - llMutex := &sync.Mutex{} - llMutex.Lock() + // create a unique key for the leaflist based on the parent entry and the leaflist name + key := struct { + parent Entry + name string + }{task.entry.GetParent(), task.importerElement.GetName()} - // try storing it or load it from leafListLock - llm, loaded := task.params.leafListLock.LoadOrStore(task.entry.SdcpbPath().ToXPath(false), llMutex) - - // if it was loaded, we need to lock the loaded mutex - if loaded { - llMutex = llm.(*sync.Mutex) - llMutex.Lock() - } - defer llMutex.Unlock() + _, loaded := task.params.leafListTracker.LoadOrStore(key, struct{}{}) var scalarArr *sdcpb.ScalarArray mustAdd := false var le *LeafEntry if loaded { + // if loaded is true, it means that another goroutine already did the first insertion and reset, + // so we just need to get the leaf list and add to it le = task.entry.GetLeafVariantEntries().GetByOwner(task.params.intentName) scalarArr = le.Value().GetLeaflistVal() } else { + // reset / create the leaf list on the first insertion le = NewLeafEntry(nil, task.params.insertFlags, task.entry) mustAdd = true scalarArr = &sdcpb.ScalarArray{Element: []*sdcpb.TypedValue{}}