diff --git a/chunk.go b/chunk.go index ae2cdd4..5399e53 100644 --- a/chunk.go +++ b/chunk.go @@ -124,7 +124,7 @@ func ErrCallback() GraphBuildCallback { return &errCallback{} } -func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, graphName string, parallel int, cb GraphBuildCallback) error { +func Chunk(ctx context.Context, sliceSize int64, parentPath string, targetPaths []string, carDir, graphName string, parallel int, cb GraphBuildCallback) error { var cumuSize int64 = 0 graphSliceCount := 0 graphFiles := make([]Finfo, 0) @@ -134,17 +134,23 @@ func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, if parallel <= 0 { return xerrors.Errorf("Unexpected! Parallel has to be greater than 0") } - if parentPath == "" { - parentPath = targetPath - } - args := []string{targetPath} - sliceTotal := GetGraphCount(args, sliceSize) + sliceTotal := GetGraphCount(targetPaths, sliceSize) if sliceTotal == 0 { log.Warn("Empty folder or file!") return nil } - files := GetFileListAsync(args) + var files chan Finfo + for _, thisPath := range targetPaths { + f := GetFileListAsync(targetPaths) + var p string + if parentPath == "" { + p = thisPath + } else { + p = parentPath + } + SetParentPathAsync(f, files, p) + } for item := range files { fileSize := item.Info.Size() switch { @@ -155,7 +161,7 @@ func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, cumuSize += fileSize graphFiles = append(graphFiles, item) // todo build ipld from graphFiles - BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), parentPath, carDir, parallel, cb) + BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), carDir, parallel, cb) fmt.Printf("cumu-size: %d\n", cumuSize) fmt.Printf(GenGraphName(graphName, graphSliceCount, sliceTotal)) fmt.Printf("=================\n") @@ -181,7 +187,7 @@ func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, }) fileSliceCount++ // todo build ipld from graphFiles - BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), parentPath, carDir, parallel, cb) + BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), carDir, parallel, cb) fmt.Printf("cumu-size: %d\n", cumuSize+firstCut) fmt.Printf(GenGraphName(graphName, graphSliceCount, sliceTotal)) fmt.Printf("=================\n") @@ -207,7 +213,7 @@ func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, fileSliceCount++ if seekEnd-seekStart == sliceSize-1 { // todo build ipld from graphFiles - BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), parentPath, carDir, parallel, cb) + BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), carDir, parallel, cb) fmt.Printf("cumu-size: %d\n", sliceSize) fmt.Printf(GenGraphName(graphName, graphSliceCount, sliceTotal)) fmt.Printf("=================\n") @@ -221,7 +227,7 @@ func Chunk(ctx context.Context, sliceSize int64, parentPath, targetPath, carDir, } if cumuSize > 0 { // todo build ipld from graphFiles - BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), parentPath, carDir, parallel, cb) + BuildIpldGraph(ctx, graphFiles, GenGraphName(graphName, graphSliceCount, sliceTotal), carDir, parallel, cb) fmt.Printf("cumu-size: %d\n", cumuSize) fmt.Printf(GenGraphName(graphName, graphSliceCount, sliceTotal)) fmt.Printf("=================\n") diff --git a/cmd/graphsplit/main.go b/cmd/graphsplit/main.go index 30d1765..0d9fd97 100644 --- a/cmd/graphsplit/main.go +++ b/cmd/graphsplit/main.go @@ -99,7 +99,7 @@ var chunkCmd = &cli.Command{ return xerrors.Errorf("Unexpected! Slice size has been set as 0") } - targetPath := c.Args().First() + targetPaths := c.Args().Slice() var cb graphsplit.GraphBuildCallback if c.Bool("calc-commp") { cb = graphsplit.CommPCallback(carDir, c.Bool("rename"), c.Bool("add-padding")) @@ -108,7 +108,7 @@ var chunkCmd = &cli.Command{ } else { cb = graphsplit.ErrCallback() } - return graphsplit.Chunk(ctx, int64(sliceSize), parentPath, targetPath, carDir, graphName, int(parallel), cb) + return graphsplit.Chunk(ctx, int64(sliceSize), parentPath, targetPaths, carDir, graphName, int(parallel), cb) }, } diff --git a/utils.go b/utils.go index 77070e4..d937d8e 100644 --- a/utils.go +++ b/utils.go @@ -40,11 +40,12 @@ const UnixfsLinksPerLevel = 1 << 10 const UnixfsChunkSize uint64 = 1 << 20 type Finfo struct { - Path string - Name string - Info os.FileInfo - SeekStart int64 - SeekEnd int64 + Path string + ParentPath string + Name string + Info os.FileInfo + SeekStart int64 + SeekEnd int64 } // file system tree node @@ -125,8 +126,8 @@ func (b *FSBuilder) getNodeByLink(ln *format.Link) (fn fsNode, err error) { return } -func BuildIpldGraph(ctx context.Context, fileList []Finfo, graphName, parentPath, carDir string, parallel int, cb GraphBuildCallback) { - node, fsDetail, err := buildIpldGraph(ctx, fileList, parentPath, carDir, parallel) +func BuildIpldGraph(ctx context.Context, fileList []Finfo, graphName, carDir string, parallel int, cb GraphBuildCallback) { + node, fsDetail, err := buildIpldGraph(ctx, fileList, carDir, parallel) if err != nil { //log.Fatal(err) cb.OnError(err) @@ -135,7 +136,7 @@ func BuildIpldGraph(ctx context.Context, fileList []Finfo, graphName, parentPath cb.OnSuccess(node, graphName, fsDetail) } -func buildIpldGraph(ctx context.Context, fileList []Finfo, parentPath, carDir string, parallel int) (ipld.Node, string, error) { +func buildIpldGraph(ctx context.Context, fileList []Finfo, carDir string, parallel int) (ipld.Node, string, error) { bs2 := bstore.NewBlockstore(dss.MutexWrap(datastore.NewMapDatastore())) dagServ := merkledag.NewDAGService(blockservice.New(bs2, offline.Exchange(bs2))) @@ -195,7 +196,7 @@ func buildIpldGraph(ctx context.Context, fileList []Finfo, parentPath, carDir st // log.Info(item.Path) // log.Infof("file name: %s, file size: %d, item size: %d, seek-start:%d, seek-end:%d", item.Name, item.Info.Size(), item.SeekEnd-item.SeekStart, item.SeekStart, item.SeekEnd) dirStr := path.Dir(item.Path) - parentPath = path.Clean(parentPath) + parentPath := path.Clean(item.ParentPath) // when parent path equal target path, and the parent path is also a file path if parentPath == path.Clean(item.Path) { dirStr = "" @@ -492,6 +493,15 @@ func GetFileListAsync(args []string) chan Finfo { return fichan } +func SetParentPathAsync(in chan Finfo, out chan Finfo, parentPath string) { + go func() { + for item := range in { + item.ParentPath = parentPath + out <- item + } + }() +} + func GetFileList(args []string) (fileList []string, err error) { fileList = make([]string, 0) for _, path := range args {