diff --git a/.gitignore b/.gitignore index 5c1baca..108f63a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.idea/ + # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a diff --git a/.travis.yml b/.travis.yml index 3777367..d98a6b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,15 @@ language: go -sudo: false +os: linux addons: - postgresql: 9.4 + postgresql: "9.4" go: - - 1.8 - - 1.9 + - "1.11" - tip cache: directories: - samples install: - - go get github.com/codegangsta/cli - - go get github.com/lib/pq - - go get github.com/kennygrant/sanitize - - go get github.com/cheggaaa/pb - - go get github.com/JensRantil/go-csv - ./download_samples.sh script: - - go install && ./test.sh - - go test + - env GO111MODULE=on go install github.com/lukasmartinelli/pgfutter && ./test.sh + - env GO111MODULE=on go test -v github.com/lukasmartinelli/pgfutter diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..280ddcc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:alpine AS DOWNLOAD + +WORKDIR /app +COPY ./download_samples.sh /app/download_samples.sh +RUN sh /app/download_samples.sh + +FROM golang:alpine + +WORKDIR /app +COPY ./ /app +ENV GO111MODULE=on + +COPY --from=DOWNLOAD /app/samples /app/samples +RUN apk update && apk add postgresql-client +RUN go install github.com/lukasmartinelli/pgfutter + +CMD ["/bin/sh", "/app/test.sh"] \ No newline at end of file diff --git a/csv.go b/csv.go index 1b05a51..14969a2 100644 --- a/csv.go +++ b/csv.go @@ -125,7 +125,7 @@ func copyCSVRows(i *Import, reader *csv.Reader, ignoreErrors bool, } func importCSV(filename string, connStr string, schema string, tableName string, ignoreErrors bool, - skipHeader bool, fields string, delimiter string, excel bool, nullDelimiter string) error { + skipHeader bool, fields string, delimiter string, excel bool, nullDelimiter string, lineTerminator string) error { db, err := connect(connStr, schema) if err != nil { @@ -138,12 +138,16 @@ func importCSV(filename string, connStr string, schema string, tableName string, // Excel 2008 and 2011 and possibly other versions uses a carriage return \r // rather than a line feed \n as a newline - if excel { - dialect.LineTerminator = "\r" - } else { - dialect.LineTerminator = "\n" - } + dialect.LineTerminator = lineTerminator + if dialect.LineTerminator == "" { + if excel { + dialect.LineTerminator = "\r" + } else { + dialect.LineTerminator = "\n" + } + } + var reader *csv.Reader var bar *pb.ProgressBar if filename != "" { diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..eb821b3 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "3" +services: + postgres: + image: postgres:latest + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + logging: + driver: none + + test: + build: + context: . + environment: + DB_USER: postgres + PGPASSWORD: secret + DB_HOST: postgres + depends_on: + - postgres \ No newline at end of file diff --git a/download_samples.sh b/download_samples.sh index cdc20d1..bac2877 100755 --- a/download_samples.sh +++ b/download_samples.sh @@ -5,7 +5,7 @@ SAMPLES_DIR="$CWD/samples" function download_json_samples() { mkdir -p $SAMPLES_DIR cd $SAMPLES_DIR - wget -nc wget -nc https://github.com/lukasmartinelli/pgfutter/releases/download/v0.1-alpha/json_sample_2015-01-01-15.json + wget -nc https://github.com/lukasmartinelli/pgfutter/releases/download/v0.1-alpha/json_sample_2015-01-01-15.json cd $CWD } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f929a9c --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/lukasmartinelli/pgfutter + +go 1.14 + +require ( + github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a + github.com/cheggaaa/pb v1.0.29 + github.com/kennygrant/sanitize v1.2.4 + github.com/lib/pq v1.8.0 + github.com/urfave/cli v1.22.4 + github.com/urfave/cli/v2 v2.2.0 + golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..af17494 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a h1:fywL1sHHCOgztiCwVQTonfyM1IR6B2lciAu3rthl3Y4= +github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a/go.mod h1:Ym6iV3U8nw0LeBLakDn42VdNggbAxPSlZH1G6dtkAO0= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pgfutter.go b/pgfutter.go index 5e1c918..142c508 100644 --- a/pgfutter.go +++ b/pgfutter.go @@ -1,24 +1,18 @@ package main import ( + "fmt" "log" "os" "path/filepath" "strings" - "github.com/codegangsta/cli" + "github.com/urfave/cli/v2" ) -func exitOnError(err error) { - log.SetFlags(0) - if err != nil { - log.Fatalln(err) - } -} - //Parse table to copy to from given filename or passed flags func parseTableName(c *cli.Context, filename string) string { - tableName := c.GlobalString("table") + tableName := c.String("table") if tableName == "" { if filename == "" { // if no filename is not set, we reading stdin @@ -33,120 +27,146 @@ func parseTableName(c *cli.Context, filename string) string { func getDataType(c *cli.Context) string { dataType := "json" - if c.GlobalBool("jsonb") { + if c.Bool("jsonb") { dataType = "jsonb" } return dataType } +func getInputFile(c *cli.Context, typ string) (string, error) { + filenames := c.Args().Slice() + if len(filenames) < 1 { + return "", fmt.Errorf("missing %s input file", typ) + } + if len(filenames) > 1 { + return "", fmt.Errorf("need exactly one %s input file, got %d. note that any flags must come before the filename", typ, len(filenames)) + } + return filenames[0], nil +} + func main() { app := cli.NewApp() app.Name = "pgfutter" app.Version = "1.2" app.Usage = "Import JSON and CSV into PostgreSQL the easy way" app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "dbname, db", + &cli.StringFlag{ + Name: "dbname", + Aliases:[]string{"db"}, Value: "postgres", Usage: "database to connect to", - EnvVar: "DB_NAME", + EnvVars: []string{"DB_NAME"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "host", Value: "localhost", Usage: "host name", - EnvVar: "DB_HOST", + EnvVars: []string{"DB_HOST"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "port", Value: "5432", Usage: "port", - EnvVar: "DB_PORT", + EnvVars: []string{"DB_PORT"}, }, - cli.StringFlag{ - Name: "username, user", + &cli.StringFlag{ + Name: "username", + Aliases:[]string{"user"}, Value: "postgres", Usage: "username", - EnvVar: "DB_USER", + EnvVars: []string{"DB_USER"}, }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "ssl", Usage: "require ssl mode", }, - cli.StringFlag{ - Name: "pass, pw", + &cli.StringFlag{ + Name: "pass", + Aliases:[]string{"pw"}, Value: "", Usage: "password", - EnvVar: "DB_PASS", + EnvVars: []string{"DB_PASS"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "schema", Value: "import", Usage: "database schema", - EnvVar: "DB_SCHEMA", + EnvVars: []string{"DB_SCHEMA"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "table", Usage: "destination table", - EnvVar: "DB_TABLE", + EnvVars: []string{"DB_TABLE"}, }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "jsonb", + Value: false, Usage: "use JSONB data type", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "ignore-errors", Usage: "halt transaction on inconsistencies", }, } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ { Name: "json", Usage: "Import newline-delimited JSON objects into database", Action: func(c *cli.Context) error { cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "", -1) - filename := c.Args().First() - - ignoreErrors := c.GlobalBool("ignore-errors") - schema := c.GlobalString("schema") + filename, err := getInputFile(c, "json") + if err != nil { + return err + } + + ignoreErrors := c.Bool("ignore-errors") + schema := c.String("schema") tableName := parseTableName(c, filename) dataType := getDataType(c) connStr := parseConnStr(c) - err := importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType) - return err + return importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType) }, }, { Name: "csv", Usage: "Import CSV into database", Flags: []cli.Flag{ - cli.BoolFlag{ + &cli.BoolFlag{ Name: "excel", + Value: false, Usage: "support problematic Excel 2008 and Excel 2011 csv line endings", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "skip-header", Usage: "skip header row", }, - cli.StringFlag{ + &cli.StringFlag{ Name: "fields", Usage: "comma separated field names if no header row", }, - cli.StringFlag{ - Name: "delimiter, d", - Value: ",", - Usage: "field delimiter", + &cli.StringFlag{ + Name: "delimiter", + Aliases: []string{"d"}, + Value: ",", + Usage: "field delimiter", }, - cli.StringFlag{ - Name: "null-delimiter, nd", - Value: "\\N", - Usage: "null delimiter", + &cli.StringFlag{ + Name: "line-terminator", + Aliases: []string{"lb", "line-break", "terminator"}, + Value: "", + Usage: "line terminator (default is newline or carriage return for excel)", }, - cli.BoolFlag{ + &cli.StringFlag{ + Name: "null-delimiter", + Aliases: []string{"nd"}, + Value: "\\N", + Usage: "null delimiter", + }, + &cli.BoolFlag{ Name: "skip-parse-delimiter", Usage: "skip parsing escape sequences in the given delimiter", }, @@ -154,10 +174,13 @@ func main() { Action: func(c *cli.Context) error { cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "", -1) - filename := c.Args().First() + filename, err := getInputFile(c, "csv") + if err != nil { + return err + } - ignoreErrors := c.GlobalBool("ignore-errors") - schema := c.GlobalString("schema") + ignoreErrors := c.Bool("ignore-errors") + schema := c.String("schema") tableName := parseTableName(c, filename) skipHeader := c.Bool("skip-header") @@ -166,12 +189,17 @@ func main() { skipParseheader := c.Bool("skip-parse-delimiter") excel := c.Bool("excel") delimiter := parseDelimiter(c.String("delimiter"), skipParseheader) + lineTerminator := c.String("line-terminator") connStr := parseConnStr(c) - err := importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel, nullDelimiter) - return err + + return importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel, nullDelimiter, lineTerminator) }, }, } - app.Run(os.Args) + err := app.Run(os.Args) + if err != nil { + log.SetFlags(0) + log.Fatal(err) + } } diff --git a/postgres.go b/postgres.go index 4b53b1b..b8bcf22 100644 --- a/postgres.go +++ b/postgres.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/codegangsta/cli" + "github.com/urfave/cli/v2" "github.com/kennygrant/sanitize" ) @@ -81,15 +81,15 @@ func postgresify(identifier string) string { //parse sql connection string from cli flags func parseConnStr(c *cli.Context) string { otherParams := "sslmode=disable connect_timeout=5" - if c.GlobalBool("ssl") { + if c.Bool("ssl") { otherParams = "sslmode=require connect_timeout=5" } return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s", - c.GlobalString("username"), - c.GlobalString("dbname"), - c.GlobalString("pass"), - c.GlobalString("host"), - c.GlobalString("port"), + c.String("username"), + c.String("dbname"), + c.String("pass"), + c.String("host"), + c.String("port"), otherParams, ) } diff --git a/test.sh b/test.sh index a78a1c7..cb0b34e 100755 --- a/test.sh +++ b/test.sh @@ -1,31 +1,32 @@ #!/bin/bash readonly CWD=$(pwd) readonly SAMPLES_DIR="$CWD/samples" +readonly DB_HOST="${DB_HOST:-localhost}" readonly DB_USER="${DB_USER:-postgres}" readonly DB_NAME="integration_test" readonly DB_SCHEMA="import" # Use public schema instead of import because of permissions function recreate_db() { - psql -U "${DB_USER}" -c "drop database if exists ${DB_NAME};" - psql -U "${DB_USER}" -c "create database ${DB_NAME};" + psql -h "${DB_HOST}" -U "${DB_USER}" -c "drop database if exists ${DB_NAME};" + psql -h "${DB_HOST}" -U "${DB_USER}" -c "create database ${DB_NAME};" } function query_counts() { local table="$1" - local counts=$(psql -U "${DB_USER}" -d "${DB_NAME}" -t -c "select count(*) from ${DB_SCHEMA}.${table}") + local counts=$(psql -h "${DB_HOST}" -U "${DB_USER}" -d "${DB_NAME}" -t -c "select count(*) from ${DB_SCHEMA}.${table}") echo "$counts" } function query_field_type() { local table="$1" - local data_type=$(psql -U "${DB_USER}" -d "${DB_NAME}" -t -c "SELECT data_type FROM information_schema.columns WHERE table_schema='${DB_SCHEMA}' AND table_name='${table}'") + local data_type=$(psql -h "${DB_HOST}" -U "${DB_USER}" -d "${DB_NAME}" -t -c "SELECT data_type FROM information_schema.columns WHERE table_schema='${DB_SCHEMA}' AND table_name='${table}'") echo "$data_type" } function import_csv_with_special_delimiter_and_trailing() { local table="csv_sample_qip12_tabdaten" local filename="$SAMPLES_DIR/csv_sample_qip12_tabdaten.csv" - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter=";" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter=";" "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -38,13 +39,13 @@ function import_csv_with_special_delimiter_and_trailing() { function import_csv_and_skip_header_row_with_custom_fields() { local table="csv_sample_qip12_tabdaten" local filename="$SAMPLES_DIR/csv_sample_qip12_tabdaten.csv" - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv --delimiter ";" "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter ";" "$filename" } function test_json() { local table=$1 local filename=$2 - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" json "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" json "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -58,7 +59,7 @@ function test_json() { function test_json_as_jsonb() { local table=$1 local filename=$2 - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --jsonb json "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" --jsonb json "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -74,8 +75,8 @@ function test_excel_csv() { local filename=$2 local delimiter=${3:-,} local general_args=${4:-} - - pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter "$delimiter" --excel + + pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter "$delimiter" --excel "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -91,7 +92,7 @@ function test_csv() { local delimiter=${3:-,} local general_args=${4:-} - pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter "$delimiter" + pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter "$delimiter" "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -111,6 +112,7 @@ recreate_db # test_csv "local_severe_wheather_warning_systems" "$SAMPLES_DIR/csv_sample_local_severe_wheather_warning_systems.csv" #TODO does not work cause quoted multiline char # test_csv "residential_permits" "$SAMPLES_DIR/csv_sample_residential_permits.csv" + test_csv "distribution_of_wealth_switzerland" "$SAMPLES_DIR/csv_sample_distribution_of_wealth_switzerland.csv" test_excel_csv "techcrunch_continental_usa" "$SAMPLES_DIR/csv_sample_techcrunch_continental_usa.csv" test_csv "employee_salaries" "$SAMPLES_DIR/csv_sample_employee_salaries.csv"