feat: Add CSV file importing handler

api_handler.go

@@ -48,4 +48,8 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
 		GetFieldsAPI(w, r, session)
 		return
 	}
+	if strings.HasPrefix(Path, "/csv_import") {
+		CSVImporterHandler(w, r, session)
+		return
+	}
 }

csv_importer.go

@@ -0,0 +1,256 @@
+package uadmin
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+	"strings"
+)
+
+// CSVImporterHandler handles CSV files
+func CSVImporterHandler(w http.ResponseWriter, r *http.Request, session *Session) {
+	if err := r.ParseMultipartForm(32 << 20); err != nil {
+        http.Error(w, err.Error(), http.StatusBadRequest)
+        return
+    }
+
+	dataJSON := r.FormValue("data")
+	if dataJSON == "" || dataJSON == "[]" {
+		Trail(ERROR, "no csv data is provided")
+        http.Error(w, "no csv data is provided", http.StatusBadRequest)
+		return
+	}
+
+	// a csv file schema is the fields in the order they are defined in the model
+	// a csv file should respect this order, the data should have two additional fields: id and language
+	// the id should be the same for all the languages (translates) of the same model instance
+	var csvFileRows []string
+	if err := json.Unmarshal([]byte(dataJSON), &csvFileRows); err != nil {
+		Trail(ERROR, err.Error())
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	modelDataMapping, err := getModelDataMapping(csvFileRows)
+	if err != nil {
+		Trail(ERROR, err.Error())
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	modelName := r.FormValue("m")
+	s, _ := getSchema(modelName)
+	fields := getFieldsList(s) // we rely on this order
+
+	csvItem := modelDataMapping[0]
+	csvItemFieldsNum := len(csvItem.LangFields[csvItem.Langs[0]])
+	if len(fields) != csvItemFieldsNum {
+		Trail(ERROR, "received wrong number of fields. Model has %d fields, received %d", len(fields), csvItemFieldsNum)
+		http.Error(w, "received wrong number of fields", http.StatusBadRequest)
+		return
+	}
+
+	for _, objectDescription := range modelDataMapping {
+		ok, err := objectExists(modelName, objectDescription, fields)
+		if err != nil {
+			Trail(ERROR, err.Error())
+			http.Error(w, "failed to process model data", http.StatusInternalServerError)
+		}
+		if ok {
+			continue
+		}
+
+		model, err := getPopulatedModel(modelName, objectDescription, fields)
+		if err != nil {
+			Trail(ERROR, "failed to process model %v", err.Error())
+			http.Error(w, "failed to process model", http.StatusBadRequest)
+			return
+		}
+		SaveRecord(model)
+	}
+
+	response := map[string]string{
+		"message": "CSV data successfully imported",
+	}
+	responseJSON, err := json.Marshal(response)
+	if err != nil {
+		http.Error(w, "failed to create response", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(responseJSON)
+}
+
+// language-values mapping for the csv file object description
+type csvEntry struct {
+	Langs      []string
+	LangFields map[string][]string // map fields to a lang
+}
+
+// returns a list of models descriptions from the provided csv file data
+// the incoming csv data should be in the following format:
+//    `idx;lang;the;rest;fields`
+//    idx — index of the object entry in the csv file, the same for all the languages
+//    lang — particular language for the object entry in the csv file
+func getModelDataMapping(csvFileRows []string) ([]csvEntry, error) {
+	if len(csvFileRows) == 0 {
+		return nil, fmt.Errorf("no csv file rows")
+	}
+
+	csvEntries := map[string]csvEntry{}
+	ids := []string{}
+	for _, row := range csvFileRows {
+		rowData := strings.Split(row, ";")
+		if len(rowData) < 3 { // expected at least 3 fields: row id, lang, model field (one or more)
+			return nil, fmt.Errorf("csv file row doesn't have any model data")
+		}
+
+		// collect all the languages and data for the same object
+		rowID := rowData[0]
+		rowLang := rowData[1]
+		if entry, ok := csvEntries[rowID]; ok {
+			// add another one language
+			entry.Langs = append(entry.Langs, rowLang)
+			entry.LangFields[rowLang] = rowData[2:]
+			csvEntries[rowID] = entry
+		} else {
+			ids = append(ids, rowID)
+			// add a new entry
+			csvEntries[rowID] = csvEntry{
+				Langs:  []string{rowLang},
+				LangFields: map[string][]string{rowLang: rowData[2:]},
+			}
+		}
+	}
+
+	data := []csvEntry{}
+	for _, id := range ids {
+		data = append(data, csvEntries[id])
+	}
+
+	return data, nil
+}
+
+type fieldDescriptor struct {
+	Name        string
+	Type        string
+	FK          bool
+	FKModelName string
+}
+
+// returns a list of a model fields in the order they are defined, marks foreign keys in the list
+func getFieldsList(s ModelSchema) []fieldDescriptor {
+	var list []fieldDescriptor
+	for _, f := range s.Fields {
+		if f.Name == "ID" { // skip the base model field
+			continue
+		}
+		fd := fieldDescriptor{
+			Name: f.Name,
+			Type: f.Type,
+		}
+		if f.Type == "fk" {
+			//TODO: this is a struct! How to find out what a struct's field is used in a csv file?..
+			// f.Name here is the name for this FK in the parent model, it's not the same as f.TypeName (FK struct's name,
+			// lower case of which is the fk's model name)
+			fd.FK = true
+			fd.FKModelName = strings.ToLower(f.TypeName)
+		}
+		list = append(list, fd)
+	}
+	return list
+}
+
+func objectExists(modelName string, objectDescription csvEntry, fieldsList []fieldDescriptor) (bool, error) {
+	lang := objectDescription.Langs[0]
+	fields := objectDescription.LangFields[lang]
+
+	var conditions []string
+	var values []interface{}
+	for idx, fieldDesc := range fieldsList {
+		if fieldDesc.Type == "fk" {
+			continue
+		}
+		conditions = append(conditions, fmt.Sprintf("%s::jsonb->>? = ?", toSnakeCase(fieldDesc.Name)))
+		values = append(values, lang, fields[idx])
+	}
+
+	var model reflect.Value
+	model, ok := NewModel(modelName, true)
+	if !ok {
+		return false, fmt.Errorf("bad model: %s", modelName)
+	}
+
+	query := strings.Join(conditions, " AND ")
+	err := Get(model.Interface(), query, values...)
+	if err != nil && err.Error() != "record not found" {
+		Trail(ERROR, "query '%s' is failed: %v", query, err)
+		return false, err
+	}
+	if err == nil && GetID(model) != 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func getPopulatedModel(modelName string, objectDescription csvEntry, fieldsList []fieldDescriptor) (reflect.Value, error) {
+	nilValue := reflect.ValueOf(nil)
+	model, ok := NewModel(modelName, true)
+	if !ok {
+		return nilValue, fmt.Errorf("bad model: %s", modelName)
+	}
+
+	for idx, fieldDesc := range fieldsList {
+		if field := model.Elem().FieldByName(fieldDesc.Name); field.IsValid() && field.CanSet() {
+			langToFieldsMap := map[string]string{} // will be marshaled to a string like `{"en":"value"}`
+			for _, lang := range objectDescription.Langs {
+				fields := objectDescription.LangFields[lang] // the values for all the fields of this model description in this lang
+				langToFieldsMap[lang] = fields[idx]
+			}
+
+			fieldsMultilangValueJSON, err := json.Marshal(langToFieldsMap)
+			if err != nil {
+				return nilValue, err
+			}
+
+			if fieldDesc.Type == "fk" {
+				m, ok := NewModel(fieldDesc.FKModelName, true)
+				if !ok {
+					return nilValue, fmt.Errorf("can't get %s model", fieldDesc.FKModelName)
+				}
+
+				// TODO: this works for one use-case only. To make it general, we need a way to
+				// pass FK FieldName with this value (see `data[idx]` above) in a csv file
+				hardcoded := "name"
+				q := fmt.Sprintf("%s::jsonb->>? = ?", toSnakeCase(hardcoded))
+				fields := langToFieldsMap[objectDescription.Langs[0]]
+				err := Get(m.Interface(), q, objectDescription.Langs[0], fields)
+				if err != nil && err.Error() != "record not found" {
+					Trail(ERROR, "query '%s' is failed: %v", q, err)
+					return nilValue, err
+				}
+				if (err != nil && err.Error() != "record not found") || GetID(m) == 0 {
+					// TODO: probably, we want to avoid creating FK object in this handler since such a struct might require data we don't have here
+					// TODO: this works for one use-case only.
+					Trail(INFO, "no record for: '%s', going to create a new one", fields)
+					hardcodedFN := "Name"
+					// foreign key model's field
+					if field := m.Elem().FieldByName(hardcodedFN); field.IsValid() && field.CanSet() {
+						field.SetString(string(fieldsMultilangValueJSON))
+					}
+					SaveRecord(m)
+				}
+
+				field.Set(m.Elem())
+				continue
+			}
+
+			field.SetString(string(fieldsMultilangValueJSON))
+		}
+	}
+
+	return model, nil
+}

csv_importer_test.go

@@ -0,0 +1,34 @@
+package uadmin
+
+import (
+	"testing"
+)
+
+func Test_getModelDataMapping(t *testing.T) {
+	rows := []string{}
+	_, err := getModelDataMapping(rows)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+
+	rows = []string{"1;en"}
+	_, err = getModelDataMapping(rows)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+
+	rows = []string{"1;en;test;fields"}
+	entries, err := getModelDataMapping(rows)
+	if err != nil {
+		t.Errorf("got unexpected error: %v", err)
+	}
+	if len(entries) != 1 {
+		t.Errorf("expected 1 entry, got %d", len(entries))
+	}
+	if entries[0].LangFields["en"][0] != "test" {
+		t.Errorf("expected 'test', got %s", entries[0].LangFields["en"][0])
+	}
+	if entries[0].LangFields["en"][1] != "fields" {
+		t.Errorf("expected 'fields', got %s", entries[0].LangFields["en"][1])
+	}
+}

get_schema.go

@@ -77,6 +77,14 @@ func getSchema(a interface{}) (s ModelSchema, ok bool) {
 			continue
 		}
 
+		// Check if the model marked as a CSV Importer
+		if t.Field(index).Anonymous && t.Field(index).Type.Name() == "Model" {
+			if strings.Contains(t.Field(index).Tag.Get("uadmin"), "csv_importer") {
+				s.CSVImporter = true
+			}
+			continue
+		}
+
 		// Initialize the field
 		f := F{
 			Translations: []translation{},

process_form.go

@@ -503,13 +503,7 @@ func processForm(modelName string, w http.ResponseWriter, r *http.Request, sessi
 	}
 
 	// Save the record
-	var saverI saver
-	saverI, ok = m.Interface().(saver)
-	if !ok {
-		Save(m.Elem().Addr().Interface())
-	} else {
-		saverI.Save()
-	}
+	SaveRecord(m)
 
 	// Save Approvals
 	for _, approval := range appList {
@@ -546,3 +540,13 @@ func processForm(modelName string, w http.ResponseWriter, r *http.Request, sessi
 	http.Redirect(w, r, newURL, http.StatusSeeOther)
 	return m
 }
+
+// SaveRecord saves the record,
+// a model is represented by a pointer (see NewModel(arg, true) function)
+func SaveRecord(model reflect.Value) {
+	if saverI, ok := model.Interface().(saver); !ok {
+		Save(model.Elem().Addr().Interface())
+	} else {
+		saverI.Save()
+	}
+}

schema.go

@@ -26,6 +26,7 @@ type ModelSchema struct {
 	ListModifier  func(*ModelSchema, *User) (string, []interface{}) `json:"-"`
 	FormTheme     string
 	ListTheme     string
+	CSVImporter   bool
 }
 
 // FieldByName returns a field from a ModelSchema by name or nil if
@@ -85,6 +86,7 @@ func (s ModelSchema) MarshalJSON() ([]byte, error) {
 		ListModifier  *string
 		FormTheme     string
 		ListTheme     string
+		CSVImporter   bool
 	}{
 		Name:          s.Name,
 		DisplayName:   s.DisplayName,
@@ -110,8 +112,9 @@ func (s ModelSchema) MarshalJSON() ([]byte, error) {
 			v := runtime.FuncForPC(reflect.ValueOf(s.ListModifier).Pointer()).Name()
 			return &v
 		}(),
-		FormTheme: s.FormTheme,
-		ListTheme: s.ListTheme,
+		FormTheme:   s.FormTheme,
+		ListTheme:   s.ListTheme,
+		CSVImporter: s.CSVImporter,
 	})
 }