Update Go version and libs

.github/workflows/build.yml

@@ -17,15 +17,15 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.29
+          version: v1.46
 
   test:
     strategy:
       matrix:
         go-version:
-          - 1.14.x
-          - 1.15.x
           - 1.16.x
+          - 1.17.x
+          - 1.18.x
         os:
           - ubuntu-latest
           - macos-latest

.github/workflows/release.yml

@@ -15,14 +15,15 @@ jobs:
           - darwin
         goarch:
           - amd64
+          - arm64
     steps:
     - uses: actions/checkout@v2
     - uses: wangyoucao577/go-release-action@master
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}
         goarch: ${{ matrix.goarch }}
-        goversion: "https://dl.google.com/go/go1.16.linux-amd64.tar.gz"
+        goversion: "https://dl.google.com/go/go1.18.linux-amd64.tar.gz"
         project_path: "."
         binary_name: "hasty"
         extra_files: LICENSE

cmd/images/gcsimport/import.go

@@ -2,8 +2,8 @@ package gcsimport
 
 import (
 	"context"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -63,7 +63,7 @@ func (i *importer) fetch(ctx context.Context, ch chan<- client.Image) {
 	defer close(ch)
 
 	log.Debug("Configure GCS client")
-	keyJSON, err := ioutil.ReadFile(i.config.CredsFile)
+	keyJSON, err := os.ReadFile(i.config.CredsFile)
 	if err != nil {
 		log.Fatalf("Unable to read credentials from file `%s`: %s", i.config.CredsFile, err)
 	}

cmd/images/s3import/import.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -16,11 +17,12 @@ import (
 	client "github.com/hasty-ai/cli/lib/hasty"
 )
 
-const batchSize int64 = 100
+const batchSize int64 = 200
 const region = "eu-central-1" // Germany, closest to Hasty, but it does not really matter
 const signTimeout = 1 * time.Hour
-const inFlight = 10        // Buffer size
-const maxImportErrors = 10 // Max consequent import errors before failing whole run
+const inFlight = 10          // Buffer size
+const maxImportErrors = 10   // Max consequent import errors before failing whole run
+const importParallelism = 10 // Number of import workers
 
 type config struct {
 	client.Config
@@ -44,17 +46,24 @@ func (i *importer) run(cmd *cobra.Command, args []string) {
 	log.Info("Perform images import from AWS S3")
 	// This context will be cancelled when all images are imported, or on problem
 	ctx, cancel := context.WithCancel(cmd.Context())
+	defer cancel()
 
 	hc := client.New(i.config.Config)
 	ch := make(chan client.Image, inFlight)
 
 	go i.fetch(ctx, ch)
-	go func() {
-		hc.ImportImages(ctx, i.config.Project, i.config.Dataset, ch)
-		cancel()
-	}()
 
-	<-ctx.Done()
+	var wg sync.WaitGroup
+	for j := 0; j < importParallelism; j++ {
+		wg.Add(1)
+		go func() {
+			hc.ImportImages(ctx, i.config.Project, i.config.Dataset, ch)
+			wg.Done()
+			// cancel()
+		}()
+	}
+
+	wg.Wait()
 }
 
 func (i *importer) fetch(ctx context.Context, ch chan<- client.Image) {
@@ -71,16 +80,19 @@ func (i *importer) fetch(ctx context.Context, ch chan<- client.Image) {
 	}
 	svc := s3.New(sess)
 
-	log.Debug("Find out S3 bucket location region")
-	resp, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{
-		Bucket: &i.config.Bucket,
-	})
-	if err != nil {
-		log.Fatalf("Unable to get S3 bucket location: %s", err)
-	}
+	/*
+		log.Debug("Find out S3 bucket location region")
+		resp, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{
+			Bucket: &i.config.Bucket,
+		})
+		if err != nil {
+			log.Fatalf("Unable to get S3 bucket location: %s", err)
+		}
+	*/
 
 	log.Debug("Re-instantiate AWS client with proper region in config")
-	cfg.Region = resp.LocationConstraint
+	// cfg.Region = resp.LocationConstraint
+	cfg.Region = aws.String("ap-southeast-2")
 	sess, err = session.NewSession(cfg)
 	if err != nil {
 		log.Fatalf("Unable to instantiate AWS API session: %s", err)

go.mod

@@ -1,17 +1,37 @@
 module github.com/hasty-ai/cli
 
-go 1.16
+go 1.18
 
 require (
-	cloud.google.com/go/storage v1.0.0
-	github.com/aws/aws-sdk-go v1.35.14
+	cloud.google.com/go/storage v1.22.0
+	github.com/aws/aws-sdk-go v1.44.11
 	github.com/hasty-ai/hasty-go v0.1.6
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	github.com/stretchr/testify v1.6.1 // indirect
-	golang.org/x/net v0.0.0-20201024042810-be3efd7ff127 // indirect
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5 // indirect
-	google.golang.org/api v0.13.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.4.0
+	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
+	google.golang.org/api v0.79.0
+)
+
+require (
+	cloud.google.com/go v0.101.1 // indirect
+	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
+	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 )