From 5295b0f0f9162ba56a963d067236f97817371c0b Mon Sep 17 00:00:00 2001 From: Duosion Date: Mon, 5 Aug 2024 03:16:15 -0500 Subject: [PATCH 1/4] Add PNG packing. --- pkg/wf/packer.go | 1 + pkg/wf/parser.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/wf/packer.go b/pkg/wf/packer.go index cff1a68..5b39900 100644 --- a/pkg/wf/packer.go +++ b/pkg/wf/packer.go @@ -70,6 +70,7 @@ func NewPacker(config *PackerConfig) (*Packer, error) { parsers := []parser{ // &amf3Parser{ext: ".action.dsl"}, // &esdlParser{&amf3Parser{ext: ".esdl"}}, + &pngParser{}, &orderedmapParser{}, // needs to be last because of ambiguous file extension } diff --git a/pkg/wf/parser.go b/pkg/wf/parser.go index 440d7dc..13cfe1a 100644 --- a/pkg/wf/parser.go +++ b/pkg/wf/parser.go @@ -193,7 +193,16 @@ func (*pngParser) matchDest(dest string, config *PackerConfig) (string, bool) { } func (*pngParser) unparse(raw []byte, config *PackerConfig) ([]byte, error) { - return nil, fmt.Errorf("unparse pngParser: not implemented") + // P N G + if raw[1] != 0x50 || raw[2] != 0x4e || raw[3] != 0x47 { + return nil, fmt.Errorf("pngParser: png header mismatch, expected: 504e47, found: %x", hex.EncodeToString(raw[1:4])) + } + + raw[1] = 0x70 + raw[2] = 0x6e + raw[3] = 0x67 + + return raw, nil } type charPngParser struct { From 440df9a6ffee1980015cdd83564c71f1b86ac7c5 Mon Sep 17 00:00:00 2001 From: Duosion Date: Mon, 5 Aug 2024 03:20:00 -0500 Subject: [PATCH 2/4] Fix PNG unparse error message. --- pkg/wf/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/wf/parser.go b/pkg/wf/parser.go index 13cfe1a..10d36e4 100644 --- a/pkg/wf/parser.go +++ b/pkg/wf/parser.go @@ -195,7 +195,7 @@ func (*pngParser) matchDest(dest string, config *PackerConfig) (string, bool) { func (*pngParser) unparse(raw []byte, config *PackerConfig) ([]byte, error) { // P N G if raw[1] != 0x50 || raw[2] != 0x4e || raw[3] != 0x47 { - return nil, fmt.Errorf("pngParser: png header mismatch, expected: 504e47, found: %x", hex.EncodeToString(raw[1:4])) + return nil, fmt.Errorf("pngUnparser: png header mismatch, expected: 504e47, found: %x", hex.EncodeToString(raw[1:4])) } raw[1] = 0x70 From 431efe85c3249b691aa19a8c3decd417289d1ab2 Mon Sep 17 00:00:00 2001 From: Duosion Date: Tue, 6 Aug 2024 23:29:55 -0500 Subject: [PATCH 3/4] Rough implementation of entities reading. --- cmd/pack.go | 3 ++ pkg/wf/extractor.go | 1 + pkg/wf/packer.go | 98 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 94 insertions(+), 8 deletions(-) diff --git a/cmd/pack.go b/cmd/pack.go index 42b9093..88ec4f6 100644 --- a/cmd/pack.go +++ b/cmd/pack.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" ) +var entitiesPathList string var packConcurrency int var packCmd = &cobra.Command{ @@ -18,6 +19,7 @@ var packCmd = &cobra.Command{ config := wf.PackerConfig{ SrcPath: filepath.Clean(args[0]), DestPath: filepath.Clean(args[1]), + Entities: filepath.Clean(entitiesPathList), Concurrency: packConcurrency, } @@ -35,5 +37,6 @@ var packCmd = &cobra.Command{ func init() { rootCmd.AddCommand(packCmd) + packCmd.Flags().StringVarP(&entitiesPathList, "entities", "e", "", "Path to csv file containing CDN entities (default \"[dest]/entities.csv\")") packCmd.Flags().IntVarP(&packConcurrency, "concurrency", "c", 5, "Maximum number of concurrent file extractions") } diff --git a/pkg/wf/extractor.go b/pkg/wf/extractor.go index f79d7bc..5aba0ba 100644 --- a/pkg/wf/extractor.go +++ b/pkg/wf/extractor.go @@ -35,6 +35,7 @@ type ExtractorConfig struct { Indent int FlattenCSV bool Eliyabot bool + Entities *map[string]AssetDirKind } // DefaultExtractorConfig generates a default configuration. diff --git a/pkg/wf/packer.go b/pkg/wf/packer.go index 5b39900..89a1a27 100644 --- a/pkg/wf/packer.go +++ b/pkg/wf/packer.go @@ -1,20 +1,28 @@ package wf import ( + "bufio" "bytes" + "errors" "fmt" "io" "log" "os" "path/filepath" + "strings" "github.com/blead/wfax/pkg/concurrency" ) +const ( + defaultEntitiesList = "entities.csv" +) + // PackerConfig is the configuration for the packer. type PackerConfig struct { SrcPath string DestPath string + Entities string Concurrency int } @@ -23,6 +31,7 @@ func DefaultPackerConfig() *PackerConfig { return &PackerConfig{ SrcPath: "", DestPath: "", + Entities: "", Concurrency: 5, } } @@ -33,6 +42,21 @@ type Packer struct { parsers []parser } +type AssetDirKind int + +// Enum values for AssetDirKind. +const ( + UPLOAD AssetDirKind = iota + MEDIUM_UPLOAD + ANDROID_UPLOAD + ANDROID_MEDIUM_UPLOAD +) + +type PathToAssetDirKindMapping struct { + Prefix string + Kind AssetDirKind +} + // NewPacker creates a new packer with the supplied configuration. // If the configuration is nil, use DefaultPackerConfig. func NewPacker(config *PackerConfig) (*Packer, error) { @@ -63,6 +87,10 @@ func NewPacker(config *PackerConfig) (*Packer, error) { } config.DestPath = filepath.Clean(config.DestPath) + if config.Entities == "" || config.Entities == "." { + config.Entities = filepath.Join(config.DestPath, defaultEntitiesList) + } + if config.Concurrency == 0 { config.Concurrency = 5 } @@ -77,7 +105,42 @@ func NewPacker(config *PackerConfig) (*Packer, error) { return &Packer{config: config, parsers: parsers}, nil } -func packFile(src string, p parser, config *PackerConfig) (bool, error) { +func (packer *Packer) readEntities() (map[string]AssetDirKind, error) { + f, err := os.Open(packer.config.Entities) + if err != nil { + // return nil if the EntitiesList does not exist. + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("readEntities: open error, path=%s, %w", packer.config.Entities, err) + } + defer f.Close() + + pathToAssetDirKind := [4]PathToAssetDirKindMapping{ + {"production/upload/", UPLOAD}, + {"production/medium_upload/", MEDIUM_UPLOAD}, + {"production/android_upload/", ANDROID_UPLOAD}, + {"production/android_medium_upload/", ANDROID_MEDIUM_UPLOAD}, + } + + el := make(map[string]AssetDirKind) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + split := strings.Split(scanner.Text(), ",") + path := split[0] + + for _, mapping := range pathToAssetDirKind { + if strings.HasPrefix(path, mapping.Prefix) { + el[strings.Replace(strings.TrimPrefix(path, mapping.Prefix), "/", "", 1)] = mapping.Kind + break + } + } + } + + return el, nil +} + +func packFile(src string, p parser, config *PackerConfig, entities *map[string]AssetDirKind) (bool, error) { path, found := p.matchDest(src, config) if !found { return false, nil @@ -88,6 +151,18 @@ func packFile(src string, p parser, config *PackerConfig) (bool, error) { return false, err } + hash := strings.Replace(strings.TrimPrefix(dest, filepath.Join(config.DestPath, dumpAssetDir)), "\\", "", 2) + kind, ok := (*entities)[hash] + if ok { + assetDirKindToString := [4]string{ + "upload", + "medium_upload", + "android_upload", + "android_medium_upload", + } + dest = filepath.Join(config.DestPath, assetDirKindToString[kind], hash[0:2], hash[2:]) + } + srcFile, err := os.Open(src) if err != nil { return false, fmt.Errorf("packFile: src open error, src=%s, dest=%s, %w", src, dest, err) @@ -120,9 +195,10 @@ func packFile(src string, p parser, config *PackerConfig) (bool, error) { } type packParams struct { - path string - parsers []parser - config *PackerConfig + path string + parsers []parser + config *PackerConfig + entities *map[string]AssetDirKind } func packPath(i *concurrency.Item[*packParams, []string]) ([]string, error) { @@ -154,7 +230,7 @@ func packPath(i *concurrency.Item[*packParams, []string]) ([]string, error) { // f is file: pack the file for _, p := range i.Data.parsers { - found, err := packFile(i.Data.path, p, i.Data.config) + found, err := packFile(i.Data.path, p, i.Data.config, i.Data.entities) if err != nil { return nil, err } @@ -171,6 +247,11 @@ func (packer *Packer) pack() error { return err } + entities, err := packer.readEntities() + if err != nil { + return err + } + f, err := os.Open(packer.config.SrcPath) if err != nil { return err @@ -196,9 +277,10 @@ func (packer *Packer) pack() error { for _, p := range i.Output { output = append(output, &concurrency.Item[*packParams, []string]{ Data: &packParams{ - path: p, - parsers: packer.parsers, - config: packer.config, + path: p, + parsers: packer.parsers, + config: packer.config, + entities: &entities, }, Output: nil, Err: nil, From 8f1a1876f0e6ec78d125be3dce172401fa69b5f3 Mon Sep 17 00:00:00 2001 From: Duosion Date: Tue, 6 Aug 2024 23:38:56 -0500 Subject: [PATCH 4/4] Remove unused variable from extractor config struct --- pkg/wf/extractor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/wf/extractor.go b/pkg/wf/extractor.go index 5aba0ba..f79d7bc 100644 --- a/pkg/wf/extractor.go +++ b/pkg/wf/extractor.go @@ -35,7 +35,6 @@ type ExtractorConfig struct { Indent int FlattenCSV bool Eliyabot bool - Entities *map[string]AssetDirKind } // DefaultExtractorConfig generates a default configuration.