Add "entities" flag to pack command.

cmd/pack.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var entitiesPathList string
 var packConcurrency int
 
 var packCmd = &cobra.Command{
@@ -18,6 +19,7 @@ var packCmd = &cobra.Command{
 		config := wf.PackerConfig{
 			SrcPath:     filepath.Clean(args[0]),
 			DestPath:    filepath.Clean(args[1]),
+			Entities:    filepath.Clean(entitiesPathList),
 			Concurrency: packConcurrency,
 		}
 
@@ -35,5 +37,6 @@ var packCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(packCmd)
+	packCmd.Flags().StringVarP(&entitiesPathList, "entities", "e", "", "Path to csv file containing CDN entities (default \"[dest]/entities.csv\")")
 	packCmd.Flags().IntVarP(&packConcurrency, "concurrency", "c", 5, "Maximum number of concurrent file extractions")
 }

pkg/wf/packer.go

@@ -1,20 +1,28 @@
 package wf
 
 import (
+	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/blead/wfax/pkg/concurrency"
 )
 
+const (
+	defaultEntitiesList = "entities.csv"
+)
+
 // PackerConfig is the configuration for the packer.
 type PackerConfig struct {
 	SrcPath     string
 	DestPath    string
+	Entities    string
 	Concurrency int
 }
 
@@ -23,6 +31,7 @@ func DefaultPackerConfig() *PackerConfig {
 	return &PackerConfig{
 		SrcPath:     "",
 		DestPath:    "",
+		Entities:    "",
 		Concurrency: 5,
 	}
 }
@@ -33,6 +42,21 @@ type Packer struct {
 	parsers []parser
 }
 
+type AssetDirKind int
+
+// Enum values for AssetDirKind.
+const (
+	UPLOAD AssetDirKind = iota
+	MEDIUM_UPLOAD
+	ANDROID_UPLOAD
+	ANDROID_MEDIUM_UPLOAD
+)
+
+type PathToAssetDirKindMapping struct {
+	Prefix string
+	Kind   AssetDirKind
+}
+
 // NewPacker creates a new packer with the supplied configuration.
 // If the configuration is nil, use DefaultPackerConfig.
 func NewPacker(config *PackerConfig) (*Packer, error) {
@@ -63,6 +87,10 @@ func NewPacker(config *PackerConfig) (*Packer, error) {
 	}
 	config.DestPath = filepath.Clean(config.DestPath)
 
+	if config.Entities == "" || config.Entities == "." {
+		config.Entities = filepath.Join(config.DestPath, defaultEntitiesList)
+	}
+
 	if config.Concurrency == 0 {
 		config.Concurrency = 5
 	}
@@ -77,7 +105,42 @@ func NewPacker(config *PackerConfig) (*Packer, error) {
 	return &Packer{config: config, parsers: parsers}, nil
 }
 
-func packFile(src string, p parser, config *PackerConfig) (bool, error) {
+func (packer *Packer) readEntities() (map[string]AssetDirKind, error) {
+	f, err := os.Open(packer.config.Entities)
+	if err != nil {
+		// return nil if the EntitiesList does not exist.
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("readEntities: open error, path=%s, %w", packer.config.Entities, err)
+	}
+	defer f.Close()
+
+	pathToAssetDirKind := [4]PathToAssetDirKindMapping{
+		{"production/upload/", UPLOAD},
+		{"production/medium_upload/", MEDIUM_UPLOAD},
+		{"production/android_upload/", ANDROID_UPLOAD},
+		{"production/android_medium_upload/", ANDROID_MEDIUM_UPLOAD},
+	}
+
+	el := make(map[string]AssetDirKind)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		split := strings.Split(scanner.Text(), ",")
+		path := split[0]
+
+		for _, mapping := range pathToAssetDirKind {
+			if strings.HasPrefix(path, mapping.Prefix) {
+				el[strings.Replace(strings.TrimPrefix(path, mapping.Prefix), "/", "", 1)] = mapping.Kind
+				break
+			}
+		}
+	}
+
+	return el, nil
+}
+
+func packFile(src string, p parser, config *PackerConfig, entities *map[string]AssetDirKind) (bool, error) {
 	path, found := p.matchDest(src, config)
 	if !found {
 		return false, nil
@@ -88,6 +151,18 @@ func packFile(src string, p parser, config *PackerConfig) (bool, error) {
 		return false, err
 	}
 
+	hash := strings.Replace(strings.TrimPrefix(dest, filepath.Join(config.DestPath, dumpAssetDir)), "\\", "", 2)
+	kind, ok := (*entities)[hash]
+	if ok {
+		assetDirKindToString := [4]string{
+			"upload",
+			"medium_upload",
+			"android_upload",
+			"android_medium_upload",
+		}
+		dest = filepath.Join(config.DestPath, assetDirKindToString[kind], hash[0:2], hash[2:])
+	}
+
 	srcFile, err := os.Open(src)
 	if err != nil {
 		return false, fmt.Errorf("packFile: src open error, src=%s, dest=%s, %w", src, dest, err)
@@ -120,9 +195,10 @@ func packFile(src string, p parser, config *PackerConfig) (bool, error) {
 }
 
 type packParams struct {
-	path    string
-	parsers []parser
-	config  *PackerConfig
+	path     string
+	parsers  []parser
+	config   *PackerConfig
+	entities *map[string]AssetDirKind
 }
 
 func packPath(i *concurrency.Item[*packParams, []string]) ([]string, error) {
@@ -154,7 +230,7 @@ func packPath(i *concurrency.Item[*packParams, []string]) ([]string, error) {
 
 	// f is file: pack the file
 	for _, p := range i.Data.parsers {
-		found, err := packFile(i.Data.path, p, i.Data.config)
+		found, err := packFile(i.Data.path, p, i.Data.config, i.Data.entities)
 		if err != nil {
 			return nil, err
 		}
@@ -171,6 +247,11 @@ func (packer *Packer) pack() error {
 		return err
 	}
 
+	entities, err := packer.readEntities()
+	if err != nil {
+		return err
+	}
+
 	f, err := os.Open(packer.config.SrcPath)
 	if err != nil {
 		return err
@@ -196,9 +277,10 @@ func (packer *Packer) pack() error {
 				for _, p := range i.Output {
 					output = append(output, &concurrency.Item[*packParams, []string]{
 						Data: &packParams{
-							path:    p,
-							parsers: packer.parsers,
-							config:  packer.config,
+							path:     p,
+							parsers:  packer.parsers,
+							config:   packer.config,
+							entities: &entities,
 						},
 						Output: nil,
 						Err:    nil,