rsql: Add manyInserter and InsertMany API

[NeilLuno]

This is an optimisation to support inserting multiple events in the same transaction.

Note that because each tuple of values has its own now(6), the timestamp of the events may be different.

Fixes #24.

[NeilLuno]

To support for i := range N.

[echarrod]

Any reason why we can't go for 1.23? If people are still using 1.22, they'd just have to use the version before this update

[NeilLuno]

No reason, just that I didn't need it.

[jrkilloran]

What about making the builder logic explicit upfront:

// makeDefaultInserter returns the default sql inserter configured via WithEventsXField options.
func makeDefaultInserter(schema eTableSchema) inserter {
	dmi := makeDefaultManyInserter(schema)
	return func(ctx context.Context, tx *sql.Tx,
		foreignID string, typ reflex.EventType, metadata []byte,
	) error {
		return dmi(ctx, tx, EventToInsert{ForeignID: foreignID, Type: typ, Metadata:  metadata})
	}
}

// makeDefaultManyInserter returns the default sql manyInserter configured via WithEventsXField options.
func makeDefaultManyInserter(schema eTableSchema) manyInserter {
	return func(ctx context.Context, tx *sql.Tx, events ...EventToInsert) error {
		if len(events) == 0 {
			return nil
		}
		q, args, err := makeInsertManyQuery(ctx, schema, events)
		if err != nil {
			return err
		}
		_, err = tx.ExecContext(ctx, q, args...)
		return errors.Wrap(err, "insert error")
	}
}

func makeInsertManyQuery(
	ctx context.Context,
	schema eTableSchema,
	events []EventToInsert,
) (query string, args []any, err error) {
	spanCtx, hasTrace := tracing.Extract(ctx)
	var traceData []byte
	if schema.traceField != "" && hasTrace {
		d, err := tracing.Marshal(spanCtx)
		if err != nil {
			return "", nil, err
		}
		traceData = d
	}

	var cols []string
	var values string
	if schema.metadataField != "" {
		if traceData != nil {
			cols, values, args = makeInsertManyFull(schema, events, traceData)
		} else {
			cols, values, args = makeInsertManyBase(schema, events)
		}
	} else if traceData != nil {
		cols, values, args = makeInsertManyTrace(schema, events, traceData)
	} else {
		cols, values, args = makeInsertManyMetadata(schema, events)
	}

	q := "insert into " + schema.name + " (" + strings.Join(cols, ", ") + ") values" + values
	return q, args, nil
}

func baseSchemaCols(schema eTableSchema) []string {
	return []string{schema.foreignIDField, schema.typeField, schema.timeField}
}

func metadataSchemaCols(schema eTableSchema) []string {
	return append(baseSchemaCols(schema), schema.metadataField)
}

func traceSchemaCols(schema eTableSchema) []string {
	return append(baseSchemaCols(schema), schema.traceField)
}

func fullSchemaCols(schema eTableSchema) []string {
	return append(metadataSchemaCols(schema), schema.traceField)
}

func makeInsertManyBase(
	schema eTableSchema,
	events []EventToInsert,
) (cols []string, values string, args []any) {
	cols = baseSchemaCols(schema)
	values = " (?, ?, now(6)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType())
	}
	return cols, values, args
}

func makeInsertManyMetadata(
	schema eTableSchema,
	events []EventToInsert,
) (cols []string, values string, args []any) {
	cols = metadataSchemaCols(schema)
	values = " (?, ?, now(6), ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), e.Metadata)
	}
	return cols, values, args
}

func makeInsertManyTrace(
	schema eTableSchema,
	events []EventToInsert,
	tracedata []byte,
) (cols []string, values string, args []any) {
	cols = traceSchemaCols(schema)
	values = " (?, ?, now(6), ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), tracedata)
	}
	return cols, values, args
}

func makeInsertManyFull(
	schema eTableSchema,
	events []EventToInsert,
	tracedata []byte,
) (cols []string, values string, args []any) {
	cols = fullSchemaCols(schema)
	values = " (?, ?, now(6), ?, ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), e.Metadata, tracedata)
	}
	return cols, values, args
}

[NeilLuno]

What about making the builder logic explicit upfront:

// makeDefaultInserter returns the default sql inserter configured via WithEventsXField options.
func makeDefaultInserter(schema eTableSchema) inserter {
	dmi := makeDefaultManyInserter(schema)
	return func(ctx context.Context, tx *sql.Tx,
		foreignID string, typ reflex.EventType, metadata []byte,
	) error {
		return dmi(ctx, tx, EventToInsert{ForeignID: foreignID, Type: typ, Metadata:  metadata})
	}
}

// makeDefaultManyInserter returns the default sql manyInserter configured via WithEventsXField options.
func makeDefaultManyInserter(schema eTableSchema) manyInserter {
	return func(ctx context.Context, tx *sql.Tx, events ...EventToInsert) error {
		if len(events) == 0 {
			return nil
		}
		q, args, err := makeInsertManyQuery(ctx, schema, events)
		if err != nil {
			return err
		}
		_, err = tx.ExecContext(ctx, q, args...)
		return errors.Wrap(err, "insert error")
	}
}

func makeInsertManyQuery(
	ctx context.Context,
	schema eTableSchema,
	events []EventToInsert,
) (query string, args []any, err error) {
	spanCtx, hasTrace := tracing.Extract(ctx)
	var traceData []byte
	if schema.traceField != "" && hasTrace {
		d, err := tracing.Marshal(spanCtx)
		if err != nil {
			return "", nil, err
		}
		traceData = d
	}

	var cols []string
	var values string
	if schema.metadataField != "" {
		if traceData != nil {
			cols, values, args = makeInsertManyFull(schema, events, traceData)
		} else {
			cols, values, args = makeInsertManyBase(schema, events)
		}
	} else if traceData != nil {
		cols, values, args = makeInsertManyTrace(schema, events, traceData)
	} else {
		cols, values, args = makeInsertManyMetadata(schema, events)
	}

	q := "insert into " + schema.name + " (" + strings.Join(cols, ", ") + ") values" + values
	return q, args, nil
}

func baseSchemaCols(schema eTableSchema) []string {
	return []string{schema.foreignIDField, schema.typeField, schema.timeField}
}

func metadataSchemaCols(schema eTableSchema) []string {
	return append(baseSchemaCols(schema), schema.metadataField)
}

func traceSchemaCols(schema eTableSchema) []string {
	return append(baseSchemaCols(schema), schema.traceField)
}

func fullSchemaCols(schema eTableSchema) []string {
	return append(metadataSchemaCols(schema), schema.traceField)
}

func makeInsertManyBase(
	schema eTableSchema,
	events []EventToInsert,
) (cols []string, values string, args []any) {
	cols = baseSchemaCols(schema)
	values = " (?, ?, now(6)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType())
	}
	return cols, values, args
}

func makeInsertManyMetadata(
	schema eTableSchema,
	events []EventToInsert,
) (cols []string, values string, args []any) {
	cols = metadataSchemaCols(schema)
	values = " (?, ?, now(6), ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), e.Metadata)
	}
	return cols, values, args
}

func makeInsertManyTrace(
	schema eTableSchema,
	events []EventToInsert,
	tracedata []byte,
) (cols []string, values string, args []any) {
	cols = traceSchemaCols(schema)
	values = " (?, ?, now(6), ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), tracedata)
	}
	return cols, values, args
}

func makeInsertManyFull(
	schema eTableSchema,
	events []EventToInsert,
	tracedata []byte,
) (cols []string, values string, args []any) {
	cols = fullSchemaCols(schema)
	values = " (?, ?, now(6), ?, ?)"
	for _, e := range events {
		args = append(args, e.ForeignID, e.Type.ReflexType(), e.Metadata, tracedata)
	}
	return cols, values, args
}

Thanks for the code suggestion. I could go either way:

Pros:

• The branches are in one place outside the loop.
• Less string wrangling.

Cons:

• More code.
• What if we add a third optional column?

(BTW values would also have to be repeated)

[edit: Looking at the code again, I wonder if we should resolve the schema even earlier - like in makeDefaultManyInserter Scratch that - the schema depends on the context]

[sonarqubecloud]

Quality Gate passed

Issues
0 New issues
0 Accepted issues

Measures
0 Security Hotspots
91.8% Coverage on New Code
0.0% Duplication on New Code

See analysis details on SonarQube Cloud

[andrewwormald]

nice I like that the singular is using the multiple inserter with just 1 event.

[andrewwormald]

Nice, I really like your solution