diff --git a/CHANGELOG.md b/CHANGELOG.md index 33894b0f..736f4621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.18.0 +## 调整 +1. 分片上传默认使用 v2 接口 + # 2.17.0 ## 新增 1. 支持智能分层 diff --git a/cmd/ali.go b/cmd/ali.go index 08b75164..6abeff77 100644 --- a/cmd/ali.go +++ b/cmd/ali.go @@ -8,8 +8,8 @@ import ( ) var aliListBucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = ali.ListBucketInfo{} - var cmd = &cobra.Command{ + info := ali.ListBucketInfo{} + cmd := &cobra.Command{ Use: "alilistbucket [Prefix] ", Short: "List all the file in the bucket of aliyun oss by prefix", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/autocompletion.go b/cmd/autocompletion.go index c000cc99..7e13cc75 100644 --- a/cmd/autocompletion.go +++ b/cmd/autocompletion.go @@ -2,9 +2,10 @@ package cmd import ( "bytes" + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/spf13/cobra" - "os" ) func completeCmdBuilder(superCmd *cobra.Command, cfg *iqshell.Config) *cobra.Command { diff --git a/cmd/bucket.go b/cmd/bucket.go index a82bb681..b60bc46d 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -10,8 +10,8 @@ import ( ) var domainsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListDomainInfo{} - var cmd = &cobra.Command{ + info := operations.ListDomainInfo{} + cmd := &cobra.Command{ Use: "domains ", Short: "Get all domains of the bucket", Run: func(cmd *cobra.Command, args []string) { @@ -27,8 +27,8 @@ var domainsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var bucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.GetBucketInfo{} - var cmd = &cobra.Command{ + info := operations.GetBucketInfo{} + cmd := &cobra.Command{ Use: "bucket ", Short: "Get bucket info", Long: `Get bucket info`, @@ -44,8 +44,8 @@ var bucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var mkBucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.CreateInfo{} - var cmd = &cobra.Command{ + info := operations.CreateInfo{} + cmd := &cobra.Command{ Use: "mkbucket ", Short: "Create a bucket in region", Long: `Create a bucket in region; @@ -65,7 +65,7 @@ The Bucket name is required to be unique within the scope of the object storage } var listBucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListInfo{ + info := operations.ListInfo{ Bucket: "", Prefix: "", Marker: "", @@ -90,7 +90,7 @@ var listBucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { OutputFileMaxSize: 0, EnableRecord: false, } - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "listbucket ", Short: "List all the files in the bucket", Long: "List all the files in the bucket to stdout if output file not specified. Each row of data information is displayed in the following order:\n Key\tSize\tHash\tPutTime\tMimeType\tFileType\tEndUser", @@ -108,8 +108,8 @@ var listBucketCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListInfo{} - var cmd = &cobra.Command{ + info := operations.ListInfo{} + cmd := &cobra.Command{ Use: "listbucket2 ", Short: "List all the files in the bucket", Long: "List all the files in the bucket to stdout if output file not specified. Each row of data information is displayed in the following order by default:\n Key\tFileSize\tHash\tPutTime\tMimeType\tFileType\tEndUser", diff --git a/cmd/cdn.go b/cmd/cdn.go index d34e6a2b..b1478cde 100644 --- a/cmd/cdn.go +++ b/cmd/cdn.go @@ -8,8 +8,8 @@ import ( ) var cdnPrefetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.PrefetchInfo{} - var cmd = &cobra.Command{ + info := operations.PrefetchInfo{} + cmd := &cobra.Command{ Use: "cdnprefetch [-i ]", Short: "Batch prefetch the urls in the url list file", Long: "Batch prefetch the urls in the url list file or from stdin if ItemListFile not specified", @@ -28,8 +28,8 @@ var cdnPrefetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var cdnRefreshCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RefreshInfo{} - var cmd = &cobra.Command{ + info := operations.RefreshInfo{} + cmd := &cobra.Command{ Use: "cdnrefresh [-i ]", Short: "Batch refresh the cdn cache by the url list file", Long: "Batch refresh the cdn cache by the url list file or from stdin if ItemListFile not specified", diff --git a/cmd/download.go b/cmd/download.go index 24912ecc..1946e328 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -107,7 +107,7 @@ have already in local disk and need to skip download or not.`, var getCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { info := operations.DownloadInfo{} - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "get ", Short: "Download a single file from bucket", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/fop.go b/cmd/fop.go index 2578bae9..e9f0305c 100644 --- a/cmd/fop.go +++ b/cmd/fop.go @@ -8,8 +8,8 @@ import ( ) var preFopStatusCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.PreFopStatusInfo{} - var cmd = &cobra.Command{ + info := operations.PreFopStatusInfo{} + cmd := &cobra.Command{ Use: "prefop ", Short: "Query the pfop status", Run: func(cmd *cobra.Command, args []string) { @@ -27,8 +27,8 @@ var preFopStatusCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var pfopCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.PfopInfo{} - var cmd = &cobra.Command{ + info := operations.PfopInfo{} + cmd := &cobra.Command{ Use: "pfop ", Short: "Issue a request to process file in bucket", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/m3u8.go b/cmd/m3u8.go index e7f5639e..7204acc8 100644 --- a/cmd/m3u8.go +++ b/cmd/m3u8.go @@ -8,8 +8,8 @@ import ( ) var m3u8ReplaceDomainCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ReplaceDomainInfo{} - var cmd = &cobra.Command{ + info := operations.ReplaceDomainInfo{} + cmd := &cobra.Command{ Use: "m3u8replace []", Short: "Replace m3u8 domain in the playlist", Run: func(cmd *cobra.Command, args []string) { @@ -33,8 +33,8 @@ var m3u8ReplaceDomainCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var m3u8DeleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.DeleteInfo{} - var cmd = &cobra.Command{ + info := operations.DeleteInfo{} + cmd := &cobra.Command{ Use: "m3u8delete ", Short: "Delete m3u8 playlist and the slices it references", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/match.go b/cmd/match.go index 33cb19e3..95720694 100644 --- a/cmd/match.go +++ b/cmd/match.go @@ -8,8 +8,8 @@ import ( ) var matchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.MatchInfo{} - var cmd = &cobra.Command{ + info := operations.MatchInfo{} + cmd := &cobra.Command{ Use: "match ", Short: "Verify that the local file matches the Qiniu cloud storage file", Run: func(cmd *cobra.Command, args []string) { @@ -30,8 +30,8 @@ var matchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchMatchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchMatchInfo{} - var cmd = &cobra.Command{ + info := operations.BatchMatchInfo{} + cmd := &cobra.Command{ Use: "batchmatch ", Short: "Batch Verify that the local file matches the Qiniu cloud storage file", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/root.go b/cmd/root.go index 60d0fd54..c16ed0b1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -67,7 +67,7 @@ var rootCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } func Execute() { - var cfg = &iqshell.Config{ + cfg := &iqshell.Config{ Document: false, DebugEnable: false, DDebugEnable: false, diff --git a/cmd/rs.go b/cmd/rs.go index 7ff90fba..7c976e0b 100644 --- a/cmd/rs.go +++ b/cmd/rs.go @@ -9,8 +9,8 @@ import ( ) var statCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.StatusInfo{} - var cmd = &cobra.Command{ + info := operations.StatusInfo{} + cmd := &cobra.Command{ Use: "stat ", Short: "Get the basic info of a remote file", Run: func(cmd *cobra.Command, args []string) { @@ -28,8 +28,8 @@ var statCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var forbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ForbiddenInfo{} - var cmd = &cobra.Command{ + info := operations.ForbiddenInfo{} + cmd := &cobra.Command{ Use: "forbidden ", Short: "Forbidden file in qiniu bucket", Long: "Forbidden object in qiniu bucket, when used with -r option, unforbidden the object", @@ -49,8 +49,8 @@ var forbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var deleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.DeleteInfo{} - var cmd = &cobra.Command{ + info := operations.DeleteInfo{} + cmd := &cobra.Command{ Use: "delete ", Short: "Delete a remote file in the bucket", Run: func(cmd *cobra.Command, args []string) { @@ -68,8 +68,8 @@ var deleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var changeLifecycleCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = &operations.ChangeLifecycleInfo{} - var cmd = &cobra.Command{ + info := &operations.ChangeLifecycleInfo{} + cmd := &cobra.Command{ Use: "chlifecycle [--to-ia-after-days ] [--to-archive-after-days ] [--to-deep-archive-after-days ] [--to-intelligent-tiering-after-days ] [--delete-after-days ]", Short: "Set the lifecycle of a file.", Long: `Set the lifecycle of a file. Lifecycle value must great than or equal to -1, unit: day. @@ -98,8 +98,8 @@ var changeLifecycleCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var deleteAfterCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.DeleteAfterInfo{} - var cmd = &cobra.Command{ + info := operations.DeleteAfterInfo{} + cmd := &cobra.Command{ Use: "expire ", Short: "Set the deleteAfterDays of a file. DeleteAfterDays:great than or equal to 0, 0: cancel expiration time, unit: day", Run: func(cmd *cobra.Command, args []string) { @@ -120,8 +120,8 @@ var deleteAfterCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var moveCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.MoveInfo{} - var cmd = &cobra.Command{ + info := operations.MoveInfo{} + cmd := &cobra.Command{ Use: "move [-k ]", Short: "Move/Rename a file and save in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -149,8 +149,8 @@ var moveCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var renameCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RenameInfo{} - var cmd = &cobra.Command{ + info := operations.RenameInfo{} + cmd := &cobra.Command{ Use: "rename ", Short: "Make a rename of a file and save in the bucket", Example: `rename A.png(bucket:bucketA key:A.png) to B.png(bucket:bucketA key:B.png): @@ -180,8 +180,8 @@ you can check if B.png has exists by: } var copyCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.CopyInfo{} - var cmd = &cobra.Command{ + info := operations.CopyInfo{} + cmd := &cobra.Command{ Use: "copy [-k ]", Short: "Make a copy of a file and save in bucket", Example: `copy A.png(bucket:bucketA key:A.png) to B.png(bucket:bucketB key:B.png): @@ -211,8 +211,8 @@ you can check if B.png has exists by: } var changeMimeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ChangeMimeInfo{} - var cmd = &cobra.Command{ + info := operations.ChangeMimeInfo{} + cmd := &cobra.Command{ Use: "chgm ", Short: "Change the mime type of a file", Example: `change mimetype of A.png(bucket:bucketA key:A.png) to image/jpeg @@ -237,8 +237,8 @@ and you can check result by command: } var changeTypeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ChangeTypeInfo{} - var cmd = &cobra.Command{ + info := operations.ChangeTypeInfo{} + cmd := &cobra.Command{ Use: "chtype ", Short: "Change the file type of a file", Long: `Change the file type of a file, file type must be one of 0, 1, 2, 3. @@ -270,8 +270,8 @@ and you can check result by command: } var restoreArCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RestoreArchiveInfo{} - var cmd = &cobra.Command{ + info := operations.RestoreArchiveInfo{} + cmd := &cobra.Command{ Use: "restorear ", Short: `Unfreeze archive file and file freeze after days, value should be between 1 and 7, include 1 and 7`, Run: func(cmd *cobra.Command, args []string) { @@ -293,8 +293,8 @@ var restoreArCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var privateUrlCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.PrivateUrlInfo{} - var cmd = &cobra.Command{ + info := operations.PrivateUrlInfo{} + cmd := &cobra.Command{ Use: "privateurl []", Short: "Create private resource access url", Run: func(cmd *cobra.Command, args []string) { @@ -312,8 +312,8 @@ var privateUrlCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var saveAsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.SaveAsInfo{} - var cmd = &cobra.Command{ + info := operations.SaveAsInfo{} + cmd := &cobra.Command{ Use: "saveas ", Short: "Create a resource access url with fop and saveas", Run: func(cmd *cobra.Command, args []string) { @@ -334,8 +334,8 @@ var saveAsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var mirrorUpdateCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.MirrorUpdateInfo{} - var cmd = &cobra.Command{ + info := operations.MirrorUpdateInfo{} + cmd := &cobra.Command{ Use: "mirrorupdate ", Short: "Fetch and update the file in bucket using mirror storage", Run: func(cmd *cobra.Command, args []string) { @@ -353,8 +353,8 @@ var mirrorUpdateCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var prefetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.MirrorUpdateInfo{} - var cmd = &cobra.Command{ + info := operations.MirrorUpdateInfo{} + cmd := &cobra.Command{ Use: "prefetch ", Short: "Fetch and update the file in bucket using mirror storage", Run: func(cmd *cobra.Command, args []string) { @@ -373,7 +373,7 @@ var prefetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { var fetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { var info operations.FetchInfo - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "fetch [-k ]", Short: "Fetch a remote resource by url and save in bucket", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/rsbatch.go b/cmd/rsbatch.go index 0a86d7b5..7c74562a 100644 --- a/cmd/rsbatch.go +++ b/cmd/rsbatch.go @@ -10,8 +10,8 @@ import ( ) var batchStatCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchStatusInfo{} - var cmd = &cobra.Command{ + info := operations.BatchStatusInfo{} + cmd := &cobra.Command{ Use: "batchstat [-i ]", Short: "Batch stat files in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -37,8 +37,8 @@ var batchStatCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchForbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchChangeStatusInfo{} - var cmd = &cobra.Command{ + info := operations.BatchChangeStatusInfo{} + cmd := &cobra.Command{ Use: "batchforbidden [-i ] [-r]", Short: "Batch forbidden files in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -64,8 +64,8 @@ var batchForbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchDeleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchDeleteInfo{} - var cmd = &cobra.Command{ + info := operations.BatchDeleteInfo{} + cmd := &cobra.Command{ Use: "batchdelete [-i ]", Short: "Batch delete files in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -82,8 +82,8 @@ var batchDeleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchChangeMimeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchChangeMimeInfo{} - var cmd = &cobra.Command{ + info := operations.BatchChangeMimeInfo{} + cmd := &cobra.Command{ Use: "batchchgm [-i ]", Short: "Batch change the mime type of files in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -100,8 +100,8 @@ var batchChangeMimeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchChangeTypeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchChangeTypeInfo{} - var cmd = &cobra.Command{ + info := operations.BatchChangeTypeInfo{} + cmd := &cobra.Command{ Use: "batchchtype [-i ]", Short: "Batch change the file type of files in bucket", Run: func(cmd *cobra.Command, args []string) { @@ -118,8 +118,8 @@ var batchChangeTypeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchRestoreArCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchRestoreArchiveInfo{} - var cmd = &cobra.Command{ + info := operations.BatchRestoreArchiveInfo{} + cmd := &cobra.Command{ Use: "batchrestorear ", Short: `Batch unfreeze archive file and file freeze after days, value should be between 1 and 7, include 1 and 7`, Run: func(cmd *cobra.Command, args []string) { @@ -139,8 +139,8 @@ var batchRestoreArCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchDeleteAfterCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchDeleteInfo{} - var cmd = &cobra.Command{ + info := operations.BatchDeleteInfo{} + cmd := &cobra.Command{ Use: "batchexpire [-i ]", Short: "Batch set the deleteAfterDays of the files in bucket. DeleteAfterDays:great than or equal to 0, 0: cancel expiration time, unit: day", Run: func(cmd *cobra.Command, args []string) { @@ -157,8 +157,8 @@ var batchDeleteAfterCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchChangeLifecycleCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchChangeLifecycleInfo{} - var cmd = &cobra.Command{ + info := operations.BatchChangeLifecycleInfo{} + cmd := &cobra.Command{ Use: "batchchlifecycle [-i ] [--to-ia-after-days ] [--to-archive-after-days ] [--to-deep-archive-after-days ] [--delete-after-days ]", Short: "Set the lifecycle of some file.", Long: `Set the lifecycle of some file. contain all file keys that need to set. one key per line. @@ -186,8 +186,8 @@ Lifecycle value must great than or equal to -1, unit: day. } var batchMoveCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchMoveInfo{} - var cmd = &cobra.Command{ + info := operations.BatchMoveInfo{} + cmd := &cobra.Command{ Use: "batchmove [-i ]", Short: "Batch move files from bucket to bucket", Run: func(cmd *cobra.Command, args []string) { @@ -208,8 +208,8 @@ var batchMoveCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchRenameCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchRenameInfo{} - var cmd = &cobra.Command{ + info := operations.BatchRenameInfo{} + cmd := &cobra.Command{ Use: "batchrename [-i ]", Short: "Batch rename files in the bucket", Run: func(cmd *cobra.Command, args []string) { @@ -227,8 +227,8 @@ var batchRenameCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchCopyCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchCopyInfo{} - var cmd = &cobra.Command{ + info := operations.BatchCopyInfo{} + cmd := &cobra.Command{ Use: "batchcopy [-i ]", Short: "Batch copy files from bucket to bucket", Run: func(cmd *cobra.Command, args []string) { @@ -249,8 +249,8 @@ var batchCopyCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchSignCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.BatchPrivateUrlInfo{} - var cmd = &cobra.Command{ + info := operations.BatchPrivateUrlInfo{} + cmd := &cobra.Command{ Use: "batchsign [-i ] [-e ]", Short: "Batch create the private url from the public url list file", Run: func(cmd *cobra.Command, args []string) { @@ -269,9 +269,9 @@ var batchSignCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var batchFetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var upHost = "" - var info = operations.BatchFetchInfo{} - var cmd = &cobra.Command{ + upHost := "" + info := operations.BatchFetchInfo{} + cmd := &cobra.Command{ Use: "batchfetch [-i ] [-c ]", Short: "Batch fetch remoteUrls and save them in qiniu Bucket", Run: func(cmd *cobra.Command, args []string) { @@ -311,40 +311,52 @@ func setBatchCmdDefaultFlags(cmd *cobra.Command, info *batch.Info) { setBatchCmdItemSeparateFlags(cmd, info) setBatchCmdForceFlags(cmd, info) } + func setBatchCmdInputFileFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().StringVarP(&info.InputFile, "input-file", "i", "", "input file, read from stdin if not set") } + func setBatchCmdForceFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().BoolVarP(&info.Force, "force", "y", false, "force mode, default false") } + func setBatchCmdWorkerCountFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().IntVarP(&info.WorkerCount, "worker", "c", 4, "worker count. 1 means the number of objects in one operation is 250 and if configured as 10 , the number of objects in one operation is 2500. This value needs to be consistent with the upper limit of Qiniu’s operation, otherwise unexpected errors will occur. Under normal circumstances you do not need to adjust this value and if you need please carefully.") } + func setBatchCmdMinWorkerCountFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().IntVarP(&info.MinWorkerCount, "min-worker", "", 1, "min worker count. 1 means the number of objects in one operation is 1000 and if configured as 3 , the number of objects in one operation is 3000. for more, please refer to worker") } + func setBatchCmdWorkerCountIncreasePeriodFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().IntVarP(&info.WorkerCountIncreasePeriod, "worker-count-increase-period", "", 60, "worker count increase period. when the worker count is too big, an overrun error will be triggered. In order to alleviate this problem, qshell will automatically reduce the worker count. In order to complete the operation as quickly as possible, qshell will periodically increase the worker count. unit: second") } + func setBatchCmdItemSeparateFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().StringVarP(&info.ItemSeparate, "sep", "F", "\t", "Separator used for split line fields, default is \\t (tab)") } + func setBatchCmdEnableRecordFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().BoolVarP(&info.EnableRecord, "enable-record", "", false, "record work progress, and do from last progress while retry") } + func setBatchCmdRecordRedoWhileErrorFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().BoolVarP(&info.RecordRedoWhileError, "record-redo-while-error", "", false, "when re-executing the command and checking the command task progress record, if a task has already been done and failed, the task will be re-executed. The default is false, and the task will not be re-executed when it detects that the task fails") } + func setBatchCmdSuccessExportFileFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().StringVarP(&info.SuccessExportFilePath, "success-list", "s", "", "specifies the file path where the successful file list is saved") } + func setBatchCmdFailExportFileFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().StringVarP(&info.FailExportFilePath, "failure-list", "e", "", "specifies the file path where the failure file list is saved") } + func setBatchCmdOverwriteFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "w", false, "overwrite mode") _ = cmd.Flags().MarkShorthandDeprecated("overwrite", "deprecated and use --overwrite instead") } + func setBatchCmdResultExportFileFlags(cmd *cobra.Command, info *batch.Info) { cmd.Flags().StringVarP(&info.ResultExportFilePath, "outfile", "o", "", "specifies the file path where the results is saved") } diff --git a/cmd/servers.go b/cmd/servers.go index 2ad76100..03dca21c 100644 --- a/cmd/servers.go +++ b/cmd/servers.go @@ -8,8 +8,8 @@ import ( ) var bucketsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListInfo{} - var cmd = &cobra.Command{ + info := operations.ListInfo{} + cmd := &cobra.Command{ Use: "buckets", Short: "Get all buckets of the account", Args: cobra.ExactArgs(0), diff --git a/cmd/share.go b/cmd/share.go index e1a6d17d..8a8e01e1 100644 --- a/cmd/share.go +++ b/cmd/share.go @@ -10,8 +10,8 @@ import ( ) var createShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.CreateShareInfo{} - var cmd = &cobra.Command{ + info := operations.CreateShareInfo{} + cmd := &cobra.Command{ Use: "create-share [kodo://]/", Short: "Share the specified directory", Run: func(cmd *cobra.Command, args []string) { @@ -38,8 +38,8 @@ var createShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var listShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListShareInfo{} - var cmd = &cobra.Command{ + info := operations.ListShareInfo{} + cmd := &cobra.Command{ Use: "share-ls ", Short: "List shared files", Run: func(cmd *cobra.Command, args []string) { @@ -58,8 +58,8 @@ var listShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var copyShareCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.CopyShareInfo{} - var cmd = &cobra.Command{ + info := operations.CopyShareInfo{} + cmd := &cobra.Command{ Use: "share-cp --to=", Short: "Copy shared files", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/tools.go b/cmd/tools.go index 908a5c71..4e07f9b4 100644 --- a/cmd/tools.go +++ b/cmd/tools.go @@ -9,8 +9,8 @@ import ( ) var rpcEncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RpcInfo{} - var cmd = &cobra.Command{ + info := operations.RpcInfo{} + cmd := &cobra.Command{ Use: "rpcencode [ [...]]", Short: "RPC Encode of qiniu", SuggestFor: []string{"rpc"}, @@ -24,8 +24,8 @@ var rpcEncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var rpcDecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RpcInfo{} - var cmd = &cobra.Command{ + info := operations.RpcInfo{} + cmd := &cobra.Command{ Use: "rpcdecode [DataToEncode...]", Short: "RPC Decode of qiniu", SuggestFor: []string{"rpc"}, @@ -39,8 +39,8 @@ var rpcDecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var base64EncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.Base64Info{} - var cmd = &cobra.Command{ + info := operations.Base64Info{} + cmd := &cobra.Command{ Use: "b64encode ", Short: "Base64 Encode, default not url safe", Long: "Base64 encode of data, url safe base64 is not turn on by default. Use --safe flag to turn it on", @@ -58,8 +58,8 @@ var base64EncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var base64DecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.Base64Info{} - var cmd = &cobra.Command{ + info := operations.Base64Info{} + cmd := &cobra.Command{ Use: "b64decode ", Short: "Base64 Decode, default nor url safe", Long: "Base64 Decode of data, urlsafe base64 is not turn on by default. Use --safe flag to turn it on", @@ -77,8 +77,8 @@ var base64DecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var ts2dCmdCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.TimestampInfo{} - var cmd = &cobra.Command{ + info := operations.TimestampInfo{} + cmd := &cobra.Command{ Use: "ts2d ", Short: "Convert timestamp in seconds to a date (TZ: Local)", Run: func(cmd *cobra.Command, args []string) { @@ -93,8 +93,8 @@ var ts2dCmdCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var tms2dCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.TimestampInfo{} - var cmd = &cobra.Command{ + info := operations.TimestampInfo{} + cmd := &cobra.Command{ Use: "tms2d ", Short: "Convert timestamp in milliseconds to a date (TZ: Local)", Run: func(cmd *cobra.Command, args []string) { @@ -109,8 +109,8 @@ var tms2dCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var tns2dCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.TimestampInfo{} - var cmd = &cobra.Command{ + info := operations.TimestampInfo{} + cmd := &cobra.Command{ Use: "tns2d ", Short: "Convert timestamp in Nanoseconds to a date (TZ: Local)", Run: func(cmd *cobra.Command, args []string) { @@ -125,8 +125,8 @@ var tns2dCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var d2tsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.TimestampInfo{} - var cmd = &cobra.Command{ + info := operations.TimestampInfo{} + cmd := &cobra.Command{ Use: "d2ts ", Short: "Create a timestamp of some seconds later, unit of timestamp is second", Run: func(cmd *cobra.Command, args []string) { @@ -141,8 +141,8 @@ var d2tsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var urlEncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.UrlInfo{} - var cmd = &cobra.Command{ + info := operations.UrlInfo{} + cmd := &cobra.Command{ Use: "urlencode ", Short: "Url Encode", Run: func(cmd *cobra.Command, args []string) { @@ -157,8 +157,8 @@ var urlEncodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var urlDecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.UrlInfo{} - var cmd = &cobra.Command{ + info := operations.UrlInfo{} + cmd := &cobra.Command{ Use: "urldecode ", Short: "Url Decode", Run: func(cmd *cobra.Command, args []string) { @@ -173,8 +173,8 @@ var urlDecodeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var etagCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.EtagInfo{} - var cmd = &cobra.Command{ + info := operations.EtagInfo{} + cmd := &cobra.Command{ Use: "qetag ", Short: "Calculate the hash of local file using the algorithm of qiniu qetag", Run: func(cmd *cobra.Command, args []string) { @@ -189,8 +189,8 @@ var etagCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var unzipCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ZipInfo{} - var cmd = &cobra.Command{ + info := operations.ZipInfo{} + cmd := &cobra.Command{ Use: "unzip []", Short: "Unzip the archive file created by the qiniu mkzip API", Long: "Unzip to current workding directory by default, use -dir to specify a directory", @@ -211,8 +211,8 @@ var unzipCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var reqIdCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ReqIdInfo{} - var cmd = &cobra.Command{ + info := operations.ReqIdInfo{} + cmd := &cobra.Command{ Use: "reqid ", Short: "Decode qiniu reqid", Run: func(cmd *cobra.Command, args []string) { @@ -227,8 +227,8 @@ var reqIdCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var IpCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.IpQueryInfo{} - var cmd = &cobra.Command{ + info := operations.IpQueryInfo{} + cmd := &cobra.Command{ Use: "ip [ [ ...]]]", Short: "Query the ip information", Run: func(cmd *cobra.Command, args []string) { @@ -241,7 +241,7 @@ var IpCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var TokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "token", Short: "Token related command", Long: "Create QBox token, Qiniu token and Upload token", @@ -260,8 +260,8 @@ var TokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var QboxTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.QBoxTokenInfo{} - var cmd = &cobra.Command{ + info := operations.QBoxTokenInfo{} + cmd := &cobra.Command{ Use: "qbox ", Short: "Create QBox token", Run: func(cmd *cobra.Command, args []string) { @@ -282,8 +282,8 @@ var QboxTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var QiniuTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.QiniuTokenInfo{} - var cmd = &cobra.Command{ + info := operations.QiniuTokenInfo{} + cmd := &cobra.Command{ Use: "qiniu ", Short: "Create Qiniu Token", Run: func(cmd *cobra.Command, args []string) { @@ -305,8 +305,8 @@ var QiniuTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var UploadTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.UploadTokenInfo{} - var cmd = &cobra.Command{ + info := operations.UploadTokenInfo{} + cmd := &cobra.Command{ Use: "upload ", Short: "Create upload token using put policy", Run: func(cmd *cobra.Command, args []string) { @@ -325,8 +325,8 @@ var UploadTokenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var dirCacheCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.DirCacheInfo{} - var cmd = &cobra.Command{ + info := operations.DirCacheInfo{} + cmd := &cobra.Command{ Use: "dircache ", Short: "Cache the directory structure of a file path", Long: "Cache the directory structure of a file path to a file, \nif not specified, cache to stdout", @@ -343,8 +343,8 @@ var dirCacheCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var funcCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.FuncCallInfo{} - var cmd = &cobra.Command{ + info := operations.FuncCallInfo{} + cmd := &cobra.Command{ Use: "func ", Short: "qshell custom function call", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/upload.go b/cmd/upload.go index b3c8c303..f0a1f241 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -71,7 +71,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().IntVar(&info.UploadConfig.WorkerCount, "worker-count", 3, "the number of concurrently uploaded parts of a single file in resumable upload") cmd.Flags().BoolVar(&info.UploadConfig.SequentialReadFile, "sequential-read-file", false, "File reading is sequential and does not involve skipping; when enabled, the uploading fragment data will be loaded into the memory. This option may increase file upload speed for mounted network filesystems.") - cmd.Flags().BoolVarP(&info.ResumableAPIV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload") + cmd.Flags().BoolVarP(&info.ResumableAPIV2, "resumable-api-v2", "", true, "use resumable upload v2 APIs to upload, default is true") cmd.Flags().Int64Var(&info.ResumableAPIV2PartSize, "resumable-api-v2-part-size", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload") cmd.Flags().BoolVar(&info.IgnoreDir, "ignore-dir", false, "ignore the dir in the dest file key") cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket") @@ -100,9 +100,9 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().IntVarP(&info.FileType, "storage", "", 0, "set storage type of file, same to --file-type") _ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage - //cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindUpIp, "bind-up-ip", "", "upload host ip to bind") - //cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindRsIp, "bind-rs-ip", "", "rs host ip to bind") - //cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindNicIp, "bind-nic-ip", "", "local network interface card to bind") + // cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindUpIp, "bind-up-ip", "", "upload host ip to bind") + // cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindRsIp, "bind-rs-ip", "", "rs host ip to bind") + // cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindNicIp, "bind-nic-ip", "", "local network interface card to bind") cmd.Flags().StringVarP(&info.EndUser, "end-user", "", "", "Owner identification") cmd.Flags().StringVarP(&info.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma") @@ -144,7 +144,7 @@ var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { }, } cmd.Flags().StringVarP(&info.SaveKey, "key", "k", "", "save as in bucket") - cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload") + cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", true, "use resumable upload v2 APIs to upload, default is true") cmd.Flags().Int64VarP(&info.ChunkSize, "resumable-api-v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, default 4M") cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "upload host") cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration") @@ -252,7 +252,7 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { cmd.Flags().StringVarP(&info.MimeType, "mimetype", "t", "", "file mime type") cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket") - cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload") + cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", true, "use resumable upload v2 APIs to upload, default is true") cmd.Flags().BoolVar(&info.SequentialReadFile, "sequential-read-file", false, "File reading is sequential and does not involve skipping; when enabled, the uploading fragment data will be loaded into the memory. This option may increase file upload speed for mounted network filesystems.") cmd.Flags().Int64VarP(&info.ChunkSize, "resumable-api-v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, default 4M") diff --git a/cmd/user.go b/cmd/user.go index 6f3ebb45..74ddbe99 100644 --- a/cmd/user.go +++ b/cmd/user.go @@ -8,9 +8,8 @@ import ( ) var accountCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var accountOver bool - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "account [ ]", Short: "Get/Set current user's AccessKey and SecretKey from local", Run: func(cmd *cobra.Command, args []string) { @@ -42,7 +41,7 @@ var accountCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { } var userCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "user", Short: "Manage local users", Args: cobra.MaximumNArgs(0), @@ -56,8 +55,8 @@ var userCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 列举本地数据库记录的账户 var userLsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ListInfo{} - var cmd = &cobra.Command{ + info := operations.ListInfo{} + cmd := &cobra.Command{ Use: "ls", Short: "List all users registered of local", Example: `qshell user ls`, @@ -74,7 +73,7 @@ var userLsCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 获取当前账户信息 var userCurrentCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "current", Short: "Get current user info from local", Example: `qshell user current`, @@ -88,8 +87,8 @@ var userCurrentCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 查询用用户是否存在本地数据库中 var userLookupCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.LookUpInfo{} - var cmd = &cobra.Command{ + info := operations.LookUpInfo{} + cmd := &cobra.Command{ Use: "lookup ", Short: "Lookup user info by user name of local", Example: `qshell user lookup `, @@ -106,8 +105,8 @@ var userLookupCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 添加用户 var userAddCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.AddInfo{} - var cmd = &cobra.Command{ + info := operations.AddInfo{} + cmd := &cobra.Command{ Use: "add", Short: "Add user info to local", Example: `qshell user add @@ -134,8 +133,8 @@ var userAddCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 切换用户 var userChCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.ChangeInfo{} - var cmd = &cobra.Command{ + info := operations.ChangeInfo{} + cmd := &cobra.Command{ Use: "cu []", Short: "Change current user with UserName", Example: `qshell user cu `, @@ -152,7 +151,7 @@ var userChCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 删除本地记录的数据库 var userCleanCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "clean", Short: "Clean account db", Long: "Remove all users of local, just remove db, current user will not remove.", @@ -167,8 +166,8 @@ var userCleanCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { // 删除用户 var userRmCmdBuilder = func(cfg *iqshell.Config) *cobra.Command { - var info = operations.RemoveInfo{} - var cmd = &cobra.Command{ + info := operations.RemoveInfo{} + cmd := &cobra.Command{ Use: "remove ", Short: "Remove user info with user name, just remove db, not influence current user.", Example: `qshell user remove `, diff --git a/cmd/version.go b/cmd/version.go index 3a329bb8..b66f13d2 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -8,7 +8,7 @@ import ( ) func versionCmdBuilder(cfg *iqshell.Config) *cobra.Command { - var cmd = &cobra.Command{ + cmd := &cobra.Command{ Use: "version", Short: "Show version", Run: func(cmd *cobra.Command, params []string) { diff --git a/cmd_test/ali_test.go b/cmd_test/ali_test.go index ecccf568..702055ae 100644 --- a/cmd_test/ali_test.go +++ b/cmd_test/ali_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestAliBucketListNoDataCenter(t *testing.T) { diff --git a/cmd_test/async_fetch_test.go b/cmd_test/async_fetch_test.go index 2e51cf9e..771bb15f 100644 --- a/cmd_test/async_fetch_test.go +++ b/cmd_test/async_fetch_test.go @@ -3,11 +3,11 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" - "github.com/qiniu/qshell/v2/iqshell/common/utils" - "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" + "github.com/qiniu/qshell/v2/iqshell/common/utils" ) func TestAsyncFetch(t *testing.T) { diff --git a/cmd_test/aws_test.go b/cmd_test/aws_test.go index 86adec40..0b4581df 100644 --- a/cmd_test/aws_test.go +++ b/cmd_test/aws_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestAwsBucketListNoBucket(t *testing.T) { diff --git a/cmd_test/bucket_list_test.go b/cmd_test/bucket_list_test.go index e65bedf4..da368b32 100644 --- a/cmd_test/bucket_list_test.go +++ b/cmd_test/bucket_list_test.go @@ -3,10 +3,11 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestBucketList(t *testing.T) { diff --git a/cmd_test/bucket_mk_test.go b/cmd_test/bucket_mk_test.go index 9a81544c..60981120 100644 --- a/cmd_test/bucket_mk_test.go +++ b/cmd_test/bucket_mk_test.go @@ -10,7 +10,6 @@ import ( ) func TestMkBucket(t *testing.T) { - _, errs := test.RunCmdWithError("mkbucket", test.Bucket, "--region", "z1", "--private") if len(errs) == 0 { t.Fatal("should return bucket exists") @@ -24,7 +23,6 @@ func TestMkBucket(t *testing.T) { } func TestMkBucketNotExistRegion(t *testing.T) { - _, errs := test.RunCmdWithError("mkbucket", test.Bucket, "--region", "z10", "--private") if len(errs) == 0 { t.Fatal("should return bucket exists") diff --git a/cmd_test/bucket_test.go b/cmd_test/bucket_test.go index d5c6c009..20beb0ce 100644 --- a/cmd_test/bucket_test.go +++ b/cmd_test/bucket_test.go @@ -3,13 +3,13 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestBucket(t *testing.T) { - ret, errs := test.RunCmdWithError("bucket", test.Bucket) if len(errs) > 0 { t.Fatal("get bucket info error:" + errs) diff --git a/cmd_test/cdn_prefetch_test.go b/cmd_test/cdn_prefetch_test.go index b139cb86..881117ab 100644 --- a/cmd_test/cdn_prefetch_test.go +++ b/cmd_test/cdn_prefetch_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestCdnPrefetch(t *testing.T) { diff --git a/cmd_test/cdn_refresh_test.go b/cmd_test/cdn_refresh_test.go index e763b71a..4476b7c0 100644 --- a/cmd_test/cdn_refresh_test.go +++ b/cmd_test/cdn_refresh_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestCdnRefreshFile(t *testing.T) { diff --git a/cmd_test/change_minetype_test.go b/cmd_test/change_minetype_test.go index ecc95bfd..9fc279ef 100644 --- a/cmd_test/change_minetype_test.go +++ b/cmd_test/change_minetype_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestMimeType(t *testing.T) { diff --git a/cmd_test/change_type_test.go b/cmd_test/change_type_test.go index 82162bd3..9f73116c 100644 --- a/cmd_test/change_type_test.go +++ b/cmd_test/change_type_test.go @@ -102,7 +102,7 @@ func TestBatchChangeType(t *testing.T) { test.RemoveFile(successLogPath) test.RemoveFile(failLogPath) - //back + // back batchConfig = "" for _, key := range test.Keys { batchConfig += key + "\t" + "1" + "\n" @@ -141,7 +141,7 @@ func TestBatchChangeTypeRecord(t *testing.T) { } defer func() { - //back + // back batchConfig = "" for _, key := range test.Keys { batchConfig += key + "\t" + "1" + "\n" diff --git a/cmd_test/delete_test.go b/cmd_test/delete_test.go index 1cf41d3f..b044e846 100644 --- a/cmd_test/delete_test.go +++ b/cmd_test/delete_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestDelete(t *testing.T) { diff --git a/cmd_test/fop_test.go b/cmd_test/fop_test.go index 400b9147..b2077b3d 100644 --- a/cmd_test/fop_test.go +++ b/cmd_test/fop_test.go @@ -9,8 +9,10 @@ import ( "github.com/qiniu/qshell/v2/cmd_test/test" ) -const fopObjectKey = "test_mv.mp4" -const fopObjectValue = "avthumb/mp4" +const ( + fopObjectKey = "test_mv.mp4" + fopObjectValue = "avthumb/mp4" +) func TestFop(t *testing.T) { result, errs := test.RunCmdWithError("pfop", test.Bucket, fopObjectKey, fopObjectValue) diff --git a/cmd_test/forbidden_test.go b/cmd_test/forbidden_test.go index 174f8d49..2b5c6ffb 100644 --- a/cmd_test/forbidden_test.go +++ b/cmd_test/forbidden_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestForbidden(t *testing.T) { @@ -84,7 +85,7 @@ func TestBatchForbidden(t *testing.T) { "--worker", "4", "-y") defer func() { - //back + // back test.RunCmdWithError("batchforbidden", test.Bucket, "-i", path, "-y", "-r") test.RemoveFile(successLogPath) @@ -122,7 +123,7 @@ func TestBatchForbiddenRecord(t *testing.T) { "-d", "-y") defer func() { - //back + // back test.RunCmdWithError("batchforbidden", test.Bucket, "-i", path, "-y", "-r") }() diff --git a/cmd_test/lifecycle_change_test.go b/cmd_test/lifecycle_change_test.go index 45d7d725..7e2d5416 100644 --- a/cmd_test/lifecycle_change_test.go +++ b/cmd_test/lifecycle_change_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestChangeLifecycle(t *testing.T) { diff --git a/cmd_test/move_test.go b/cmd_test/move_test.go index 792f6517..594e86a1 100644 --- a/cmd_test/move_test.go +++ b/cmd_test/move_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestMove(t *testing.T) { diff --git a/cmd_test/qshell_test.go b/cmd_test/qshell_test.go index 512f8010..9b8b083e 100644 --- a/cmd_test/qshell_test.go +++ b/cmd_test/qshell_test.go @@ -1,9 +1,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestQShellDocument01(t *testing.T) { diff --git a/cmd_test/rename_test.go b/cmd_test/rename_test.go index faec184d..370f399b 100644 --- a/cmd_test/rename_test.go +++ b/cmd_test/rename_test.go @@ -4,10 +4,11 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestRename(t *testing.T) { diff --git a/cmd_test/restore_archive_test.go b/cmd_test/restore_archive_test.go index 3bdbdb65..ad4e31ec 100644 --- a/cmd_test/restore_archive_test.go +++ b/cmd_test/restore_archive_test.go @@ -4,16 +4,15 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "path/filepath" "strings" "testing" -) -var ( - restoreKey = "restore_key.json" + "github.com/qiniu/qshell/v2/cmd_test/test" ) +var restoreKey = "restore_key.json" + func TestRestoreArchive(t *testing.T) { copyFile(t, test.OriginKeys[0], restoreKey) changeType(t, restoreKey, "2") diff --git a/cmd_test/servers_test.go b/cmd_test/servers_test.go index 83633702..50497ab9 100644 --- a/cmd_test/servers_test.go +++ b/cmd_test/servers_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestBuckets(t *testing.T) { diff --git a/cmd_test/status_test.go b/cmd_test/status_test.go index 004352b6..ab2283a7 100644 --- a/cmd_test/status_test.go +++ b/cmd_test/status_test.go @@ -18,9 +18,11 @@ func TestStatus(t *testing.T) { t.Fail() } - items := []string{"Bucket", "Etag", "MD5", "Fsize", "PutTime", + items := []string{ + "Bucket", "Etag", "MD5", "Fsize", "PutTime", "MimeType", "Status", "Expiration", "TransitionToIA", "TransitionToArchive", "TransitionToDeepArchive", - "FileType"} + "FileType", + } for _, item := range items { if !strings.Contains(result, item) { t.Fatal("no ", item) diff --git a/cmd_test/test/file.go b/cmd_test/test/file.go index 7347e432..32a03e89 100644 --- a/cmd_test/test/file.go +++ b/cmd_test/test/file.go @@ -2,11 +2,12 @@ package test import ( "fmt" - "github.com/mitchellh/go-homedir" "io" "io/ioutil" "os" "path/filepath" + + "github.com/mitchellh/go-homedir" ) func RootPath() (string, error) { @@ -73,7 +74,7 @@ func CreateFileWithContent(fileName, content string) (string, error) { filePath = filepath.Join(filePath, fileName) _ = RemoveFile(filePath) - if f, OErr := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600); OErr != nil { + if f, OErr := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600); OErr != nil { return "", OErr } else { _, err = f.Write([]byte(content)) @@ -110,7 +111,7 @@ func CreateTempFile(size int) (string, error) { return "", err } - f, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + f, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600) if err != nil { return "", err } diff --git a/cmd_test/test/flow.go b/cmd_test/test/flow.go index f04ca886..9fe594e3 100644 --- a/cmd_test/test/flow.go +++ b/cmd_test/test/flow.go @@ -2,11 +2,12 @@ package test import ( "fmt" + "os" + "os/exec" + "github.com/qiniu/qshell/v2/cmd" "github.com/qiniu/qshell/v2/docs" "github.com/qiniu/qshell/v2/iqshell/common/data" - "os" - "os/exec" ) type testFlow struct { diff --git a/cmd_test/tools_test.go b/cmd_test/tools_test.go index bdf3d4c3..1018760f 100644 --- a/cmd_test/tools_test.go +++ b/cmd_test/tools_test.go @@ -4,11 +4,12 @@ package cmd import ( "fmt" - "github.com/qiniu/qshell/v2/cmd_test/test" "strconv" "strings" "testing" "time" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) const ( diff --git a/cmd_test/upload_resume_test.go b/cmd_test/upload_resume_test.go index 4a34263a..75fb89ae 100644 --- a/cmd_test/upload_resume_test.go +++ b/cmd_test/upload_resume_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) // ------------------- v1 ----------------------- diff --git a/cmd_test/upload_sync_test.go b/cmd_test/upload_sync_test.go index 0e0dbec5..0a1e5284 100644 --- a/cmd_test/upload_sync_test.go +++ b/cmd_test/upload_sync_test.go @@ -3,9 +3,10 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" "testing" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func TestSyncV1(t *testing.T) { diff --git a/cmd_test/user.go b/cmd_test/user.go index ab6ae37a..f8dcd20e 100644 --- a/cmd_test/user.go +++ b/cmd_test/user.go @@ -10,8 +10,10 @@ import ( "github.com/qiniu/qshell/v2/cmd_test/test" ) -var accessKey = test.AccessKey -var secretKey = test.SecretKey +var ( + accessKey = test.AccessKey + secretKey = test.SecretKey +) func TestUser(t *testing.T) { defer func() { diff --git a/cmd_test/user_cmd_execute.go b/cmd_test/user_cmd_execute.go index ed8ef391..750b56a8 100644 --- a/cmd_test/user_cmd_execute.go +++ b/cmd_test/user_cmd_execute.go @@ -3,8 +3,9 @@ package cmd import ( - "github.com/qiniu/qshell/v2/cmd_test/test" "strings" + + "github.com/qiniu/qshell/v2/cmd_test/test" ) func addUser(userName, ak, sk string) (success bool, errorMsg string) { diff --git a/docs/document.go b/docs/document.go index c3468874..4a9fa717 100644 --- a/docs/document.go +++ b/docs/document.go @@ -3,12 +3,13 @@ package docs import ( _ "embed" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/utils" "io" "os" "os/exec" "strings" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/utils" ) type ShowMethod int @@ -18,8 +19,10 @@ const ( ShowMethodStdOut ShowMethod = 2 ) -var stdout io.Writer = os.Stdout -var showMethod = ShowMethodLess +var ( + stdout io.Writer = os.Stdout + showMethod = ShowMethodLess +) func SetStdout(o io.Writer) { stdout = o diff --git a/docs/qupload.md b/docs/qupload.md index 9bc465da..0c6d2e7a 100644 --- a/docs/qupload.md +++ b/docs/qupload.md @@ -56,7 +56,7 @@ $ qshell qupload --doc "log_rotate" : 10, "log_stdout" : true, "file_type" : 0, - "resumable_api_v2" : false, + "resumable_api_v2" : true, "resumable_api_v2_part_size" : 4194304, "uploading_acceleration" : true } @@ -83,7 +83,7 @@ $ qshell qupload --doc - log_stdout:上传日志是否同时输出一份到标准终端,默认为 `true`。 【可选】 - file_type:文件存储类型;0:标准存储 1:低频存储 2:归档存储 3:深度归档 4:归档直读存储 5:智能分层存储;默认为 0。 【可选】 - delete_on_success:上传成功的文件,同时删除本地文件,以达到节约磁盘的目的,比如日志归档的场景,默认为 `false`,如果需要开启功能,设置为 `true` 即可。【可选】 -- resumable_api_v2:使用分片 V2 进行上传,默认为 `false` 使用分片 V1 。【可选】 +- resumable_api_v2:使用分片 V2 进行上传,默认为 `true` 使用分片 V2 。【可选】 - resumable_api_v2_part_size:使用分片 V2 进行上传时定制分片大小,默认 4194304(4M) 。【可选】 - uploading_acceleration:启用上传加速。【可选】 - put_threshold:上传阈值,上传文件大小超过此值会使用分片上传,不超过使用表单上传;单位:B,默认为 8388608(8M) 。【可选】 diff --git a/docs/rput.md b/docs/rput.md index 11967200..6e54ddf9 100644 --- a/docs/rput.md +++ b/docs/rput.md @@ -35,7 +35,7 @@ $ qshell rput --doc - --overwrite:是否覆盖空间已有文件,默认为 `false`。 【可选】 - -t/--mimetype:指定文件的 MimeType 。【可选】 - --file-type:文件存储类型;0:标准存储 1:低频存储 2:归档存储 3:深度归档 4:归档直读存储 5:智能分层存储;默认为 0。 【可选】 -- --resumable-api-v2:使用分片上传 API V2 进行上传,默认为 `false`, 使用 V1 上传。【可选】 +- --resumable-api-v2:使用分片上传 API V2 进行上传,默认为 `true`, 使用 V2 上传。【可选】 - --resumable-api-v2-part-size:使用分片上传 API V2 进行上传时的分片大小,默认为 4M 。【可选】 - --sequential-read-file: 文件读为顺序读,不涉及跳读;开启后,上传中的分片数据会被加载至内存。此选项可能会增加挂载网络文件系统的文件上传速度。默认是:false。【可选】 - -l/--callback-urls:上传回调地址,可以指定多个地址,以逗号分开。【可选】 @@ -64,10 +64,10 @@ $ qshell rput --doc 1 上传本地文件 `/Users/jemy/Documents/qiniu.mp4` 到空间 `if-pbl` 里面。 ``` // 使用使用分片上传 API V1 -$ qshell rput if-pbl qiniu.mp4 /Users/jemy/Documents/qiniu.mp4 +$ qshell rput if-pbl qiniu.mp4 /Users/jemy/Documents/qiniu.mp4 --resumable-api-v2=false // 使用使用分片上传 API V2 -$ qshell rput if-pbl qiniu.mp4 /Users/jemy/Documents/qiniu.mp4 --resumable-api-v2 +$ qshell rput if-pbl qiniu.mp4 /Users/jemy/Documents/qiniu.mp4 ``` 2 上传本地文件 `/Users/jemy/Documents/qiniu.mp4` 到空间 `if-pbl` 里面,带前缀 `2015/01/18/`,并且指定 `MimeType` 参数为 `video/mp4`。 diff --git a/docs/sync.md b/docs/sync.md index b61765c0..3660179c 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -34,7 +34,7 @@ $ qshell sync --doc - -k/--key:该资源保存在空间中的 key,不配置时使用资源 Url 中文件名作为存储的 key。 【可选】 - -u/--uphost:上传入口的 IP 地址,一般在大文件的情况下,可以指定上传入口的 IP 来减少 DNS 环节,提升同步速度。 【可选】 - --file-type:文件存储类型,0:标准存储 1:低频存储 2:归档存储 3:深度归档 4:归档直读存储 5:智能分层存储;默认为 0【可选】 -- --resumable-api-v2:使用分片 v2 进行上传;默认使用 v1。 【可选】 +- --resumable-api-v2:使用分片 v2 进行上传,默认为 `true`;默认使用 v2。 【可选】 - --resumable-api-v2-part-size:使用分片上传 API V2 进行上传时的分片大小,默认为 4M 。【可选】 - --overwrite:是否覆盖空间已有文件,默认为 `false`。 【可选】 - -l/--callback-urls:上传回调地址,可以指定多个地址,以逗号分开。【可选】 @@ -83,5 +83,5 @@ $ dig up-as0.qiniu.com # 示例 使用分片 v2 抓取一个资源并以指定的文件名保存在七牛的空间里面: ``` -$ qshell sync http://if-pbl.qiniudn.com/test_big_movie.mp4 if-pbl test.mp4 --resumable-api-v2 +$ qshell sync http://if-pbl.qiniudn.com/test_big_movie.mp4 if-pbl test.mp4 ``` diff --git a/go.mod b/go.mod index 98330dcd..4d643007 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/astaxie/beego v1.12.3 github.com/aws/aws-sdk-go v1.37.31 github.com/mitchellh/go-homedir v1.1.0 - github.com/qiniu/go-sdk/v7 v7.25.5 + github.com/qiniu/go-sdk/v7 v7.25.6 github.com/schollz/progressbar/v3 v3.8.6 github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.1.3 diff --git a/go.sum b/go.sum index 11e57485..aba5df0d 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.25.5 h1:BZAZhrYC7vrw9NPnNbFi1K8xucoz29s7sEGWap77i2Q= -github.com/qiniu/go-sdk/v7 v7.25.5/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peqTowyTO8o= +github.com/qiniu/go-sdk/v7 v7.25.6 h1:89KQX16Bv2x7MxhwpzWGGvQBOPIlGpAcnPQyfS3tRok= +github.com/qiniu/go-sdk/v7 v7.25.6/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peqTowyTO8o= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/iqshell/ali/list_bucket.go b/iqshell/ali/list_bucket.go index 2dfe3be5..b4c2c136 100644 --- a/iqshell/ali/list_bucket.go +++ b/iqshell/ali/list_bucket.go @@ -3,12 +3,13 @@ package ali import ( "bufio" "fmt" + "os" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "os" ) type ListBucketInfo struct { @@ -48,7 +49,7 @@ func ListBucket(cfg *iqshell.Config, info ListBucketInfo) { return } - //open result file + // open result file fp, err := os.Create(info.SaveToFile) if err != nil { log.Error("create file error:", err) diff --git a/iqshell/aws/fetch.go b/iqshell/aws/fetch.go index 865baa3e..3a113655 100644 --- a/iqshell/aws/fetch.go +++ b/iqshell/aws/fetch.go @@ -2,6 +2,9 @@ package aws import ( "fmt" + "path/filepath" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/qiniu/qshell/v2/iqshell" @@ -14,8 +17,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/workspace" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" - "time" ) type FetchInfo struct { @@ -68,7 +69,6 @@ func Fetch(cfg *iqshell.Config, info FetchInfo) { FailExportFilePath: info.BatchInfo.FailExportFilePath, OverwriteExportFilePath: info.BatchInfo.OverwriteExportFilePath, }) - if err != nil { log.ErrorF("get export error:%v", err) data.SetCmdStatusError() @@ -169,7 +169,6 @@ func Fetch(cfg *iqshell.Config, info FetchInfo) { exporter.Fail().ExportF("%s%s%v", work.Data, flow.ErrorSeparate, err) log.DebugF("Skip line:%s because:%v", work.Data, err) } - }). OnWorkSuccess(func(workInfo *flow.WorkInfo, result flow.Result) { metric.AddCurrentCount(1) diff --git a/iqshell/aws/list_bucket.go b/iqshell/aws/list_bucket.go index 79adfc80..679d3125 100644 --- a/iqshell/aws/list_bucket.go +++ b/iqshell/aws/list_bucket.go @@ -1,6 +1,8 @@ package aws import ( + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -10,7 +12,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "strings" ) type ListBucketInfo struct { diff --git a/iqshell/cdn/operations/prefetch.go b/iqshell/cdn/operations/prefetch.go index b23bd17d..d1844dcd 100644 --- a/iqshell/cdn/operations/prefetch.go +++ b/iqshell/cdn/operations/prefetch.go @@ -1,13 +1,14 @@ package operations import ( + "strings" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/cdn" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/flow" "github.com/qiniu/qshell/v2/iqshell/common/log" - "strings" ) type PrefetchInfo struct { @@ -78,7 +79,6 @@ func Prefetch(cfg *iqshell.Config, info PrefetchInfo) { } func prefetchWithQps(urlsToPrefetch []string) { - waiterIfNeeded() log.DebugF("cdnPrefetch, url size: %d", len(urlsToPrefetch)) diff --git a/iqshell/cdn/operations/refresh.go b/iqshell/cdn/operations/refresh.go index 0a834e54..bf42740d 100644 --- a/iqshell/cdn/operations/refresh.go +++ b/iqshell/cdn/operations/refresh.go @@ -1,13 +1,14 @@ package operations import ( + "strings" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/cdn" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/flow" "github.com/qiniu/qshell/v2/iqshell/common/log" - "strings" ) type RefreshInfo struct { @@ -23,7 +24,6 @@ func (info *RefreshInfo) Check() *data.CodeError { // Refresh 【cdnrefresh】刷新所有CDN节点 func Refresh(cfg *iqshell.Config, info RefreshInfo) { - if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{ Checker: &info, }); !shouldContinue { @@ -72,7 +72,7 @@ func Refresh(cfg *iqshell.Config, info RefreshInfo) { } } - //check final items + // check final items if len(itemsToRefresh) > 0 { refreshWithQps(info, itemsToRefresh, true) } diff --git a/iqshell/common/account/account.go b/iqshell/common/account/account.go index 947fb667..3b4a646f 100644 --- a/iqshell/common/account/account.go +++ b/iqshell/common/account/account.go @@ -2,9 +2,10 @@ package account import ( "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "strings" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/go-sdk/v7/auth/qbox" ) diff --git a/iqshell/common/account/actions.go b/iqshell/common/account/actions.go index 33faedde..7f37c3d2 100644 --- a/iqshell/common/account/actions.go +++ b/iqshell/common/account/actions.go @@ -17,14 +17,14 @@ import ( // 保存账户信息到账户文件中 func SetAccountToLocalFile(acc Account) (err *data.CodeError) { - accountFh, openErr := os.OpenFile(info.AccountPath, os.O_CREATE|os.O_RDWR, 0600) + accountFh, openErr := os.OpenFile(info.AccountPath, os.O_CREATE|os.O_RDWR, 0o600) if openErr != nil { err = data.NewEmptyError().AppendDescF("Open account file error: %s", openErr) return } defer accountFh.Close() - oldAccountFh, openErr := os.OpenFile(info.OldAccountPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + oldAccountFh, openErr := os.OpenFile(info.OldAccountPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if openErr != nil { err = data.NewEmptyError().AppendDescF("Open account file error: %s", openErr) return @@ -97,7 +97,7 @@ func SaveToDB(acc Account, accountOver bool) (err *data.CodeError) { } func getAccount(pt string) (account Account, err *data.CodeError) { - accountFh, openErr := os.OpenFile(info.AccountPath, os.O_RDWR, 0600) + accountFh, openErr := os.OpenFile(info.AccountPath, os.O_RDWR, 0o600) if openErr != nil { err = data.NewEmptyError().AppendDescF("Get account error, %s, please use `account` to set Id and SecretKey first", openErr) return @@ -240,7 +240,6 @@ func ChUser(userName string) (name string, err *data.CodeError) { // 获取用户列表 func GetUsers() (ret []*Account, err *data.CodeError) { - db, gErr := leveldb.OpenFile(info.AccountDBPath, nil) if gErr != nil { err = data.NewEmptyError().AppendDescF("open db: %v", err) diff --git a/iqshell/common/account/operations/delete.go b/iqshell/common/account/operations/delete.go index 3518bf0d..7862a65a 100644 --- a/iqshell/common/account/operations/delete.go +++ b/iqshell/common/account/operations/delete.go @@ -8,8 +8,7 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/log" ) -type CleanInfo struct { -} +type CleanInfo struct{} func (info *CleanInfo) Check() *data.CodeError { return nil diff --git a/iqshell/common/account/operations/query.go b/iqshell/common/account/operations/query.go index f417a890..964b0572 100644 --- a/iqshell/common/account/operations/query.go +++ b/iqshell/common/account/operations/query.go @@ -8,8 +8,7 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/log" ) -type UserInfo struct { -} +type UserInfo struct{} func (info *UserInfo) Check() *data.CodeError { return nil @@ -45,12 +44,12 @@ func List(cfg *iqshell.Config, info ListInfo) { for index, acc := range accounts { if info.OnlyListName { - log.AlertF(acc.Name) + log.Alert(acc.Name) } else { if index > 0 { log.Alert(" ") } - log.AlertF(acc.String()) + log.Alert(acc.String()) } } } @@ -69,7 +68,7 @@ func Current(cfg *iqshell.Config) { data.SetCmdStatusError() return } - log.AlertF(acc.String()) + log.Alert(acc.String()) } // LookUpInfo 查找某个用户 @@ -98,7 +97,7 @@ func LookUp(cfg *iqshell.Config, info LookUpInfo) { return } for _, acc := range accounts { - log.AlertF(acc.String()) + log.Alert(acc.String()) log.Alert("") } diff --git a/iqshell/common/config/operation.go b/iqshell/common/config/operation.go index c3e4a8b7..fec88309 100644 --- a/iqshell/common/config/operation.go +++ b/iqshell/common/config/operation.go @@ -4,9 +4,10 @@ import ( "bufio" "encoding/json" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "io/ioutil" "os" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) func NewConfigWithPath(path string) (*Config, *data.CodeError) { @@ -32,7 +33,7 @@ func NewConfigWithPath(path string) (*Config, *data.CodeError) { } func (c *Config) UpdateToLocal(path string) *data.CodeError { - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0666) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0o666) if err != nil || file == nil { return data.NewEmptyError().AppendError(err) } diff --git a/iqshell/common/config/viper.go b/iqshell/common/config/viper.go index 3697e411..8eb6632e 100644 --- a/iqshell/common/config/viper.go +++ b/iqshell/common/config/viper.go @@ -47,7 +47,7 @@ func getValueFromLocal(vipers []*viper.Viper, localKey []string, f func(value in return } - var stop = false + stop := false var value interface{} for _, key := range localKey { for _, vip := range vipers { diff --git a/iqshell/common/data/data_test.go b/iqshell/common/data/data_test.go index 0c636f0d..373678d2 100644 --- a/iqshell/common/data/data_test.go +++ b/iqshell/common/data/data_test.go @@ -13,7 +13,6 @@ type Data struct { } func TestData(t *testing.T) { - d := Data{ Can: NewBool(true), Age: NewInt(10), diff --git a/iqshell/common/data/error.go b/iqshell/common/data/error.go index c7f0fd1b..8d379f9d 100644 --- a/iqshell/common/data/error.go +++ b/iqshell/common/data/error.go @@ -15,9 +15,7 @@ var ( ErrorCodeAlreadyDone = -15000 ) -var ( - CancelError = NewError(ErrorCodeCancel, "user cancel") -) +var CancelError = NewError(ErrorCodeCancel, "user cancel") func ConvertError(err error) *CodeError { if err == nil { diff --git a/iqshell/common/data/std.go b/iqshell/common/data/std.go index f2b4e636..43989d2a 100644 --- a/iqshell/common/data/std.go +++ b/iqshell/common/data/std.go @@ -5,8 +5,10 @@ import ( "os" ) -var stdout io.WriteCloser = os.Stdout -var stderr io.WriteCloser = os.Stderr +var ( + stdout io.WriteCloser = os.Stdout + stderr io.WriteCloser = os.Stderr +) func Stdout() io.WriteCloser { return stdout diff --git a/iqshell/common/db/db.go b/iqshell/common/db/db.go index ac908f74..46e6175b 100644 --- a/iqshell/common/db/db.go +++ b/iqshell/common/db/db.go @@ -1,13 +1,14 @@ package db import ( + "os" + "path/filepath" + "sync" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" - "os" - "path/filepath" - "sync" ) type DB struct { @@ -15,8 +16,10 @@ type DB struct { db *leveldb.DB } -var dbMap map[string]*DB -var dbMapLock sync.Mutex +var ( + dbMap map[string]*DB + dbMapLock sync.Mutex +) func OpenDB(filePath string) (*DB, *data.CodeError) { dbMapLock.Lock() diff --git a/iqshell/common/export/export.go b/iqshell/common/export/export.go index b79d1f89..f9ad4de1 100644 --- a/iqshell/common/export/export.go +++ b/iqshell/common/export/export.go @@ -3,9 +3,10 @@ package export import ( "bufio" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "os" "sync" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) type Exporter interface { diff --git a/iqshell/common/file/rotate_file.go b/iqshell/common/file/rotate_file.go index c18f2e84..5ad54f8a 100644 --- a/iqshell/common/file/rotate_file.go +++ b/iqshell/common/file/rotate_file.go @@ -2,8 +2,6 @@ package file import ( "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/utils" "io" "io/fs" "os" @@ -11,6 +9,9 @@ import ( "strconv" "strings" "sync" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/utils" ) const ( @@ -51,7 +52,6 @@ func RotateOptionOnOpenFile(f func(filename string)) RotateOption { } func NewRotateFile(name string, options ...RotateOption) (io.WriteCloser, *data.CodeError) { - if n, aErr := filepath.Abs(name); aErr != nil { return nil, data.ConvertError(aErr) } else { @@ -150,7 +150,6 @@ func (r *rotateFile) writeByRotateWithLock(p []byte) (n int, err error) { } func (r *rotateFile) writeByRotate(p []byte) (n int, err error) { - // 不滚动 if r.maxLine <= 0 && r.maxSize <= 0 { return r.file.Write(p) @@ -208,7 +207,7 @@ func (r *rotateFile) writeLine(isNew bool, line string) (n int, err error) { } func (r *rotateFile) createFile() error { - if mErr := os.MkdirAll(r.fileDir, 0766); mErr != nil { + if mErr := os.MkdirAll(r.fileDir, 0o766); mErr != nil { return mErr } @@ -233,7 +232,7 @@ func (r *rotateFile) createFile() error { } newFileName = filepath.Join(r.fileDir, newFileName) - if file, err := os.OpenFile(newFileName, flag, 0666); err != nil { + if file, err := os.OpenFile(newFileName, flag, 0o666); err != nil { return err } else { r.file = file @@ -271,7 +270,6 @@ func (r *rotateFile) createFile() error { } func (r *rotateFile) getFileIndex() (index int, err *data.CodeError) { - // 找到最新的文件 wErr := filepath.WalkDir(r.fileDir, func(path string, d fs.DirEntry, err error) error { if d.IsDir() { diff --git a/iqshell/common/file/rotate_file_test.go b/iqshell/common/file/rotate_file_test.go index 6b282034..3473650e 100644 --- a/iqshell/common/file/rotate_file_test.go +++ b/iqshell/common/file/rotate_file_test.go @@ -5,11 +5,12 @@ package file import ( "errors" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/utils" "io/fs" "os" "path/filepath" "testing" + + "github.com/qiniu/qshell/v2/iqshell/common/utils" ) func testDir() (string, error) { diff --git a/iqshell/common/flow/code_verification.go b/iqshell/common/flow/code_verification.go index 79a90311..797f60e1 100644 --- a/iqshell/common/flow/code_verification.go +++ b/iqshell/common/flow/code_verification.go @@ -2,6 +2,7 @@ package flow import ( "fmt" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" diff --git a/iqshell/common/flow/overseer_local_db.go b/iqshell/common/flow/overseer_local_db.go index ff49901b..f22e603c 100644 --- a/iqshell/common/flow/overseer_local_db.go +++ b/iqshell/common/flow/overseer_local_db.go @@ -2,6 +2,7 @@ package flow import ( "encoding/json" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/recorder" ) diff --git a/iqshell/common/flow/redo.go b/iqshell/common/flow/redo.go index 5c93c7c2..4af4d088 100644 --- a/iqshell/common/flow/redo.go +++ b/iqshell/common/flow/redo.go @@ -3,7 +3,6 @@ package flow import "github.com/qiniu/qshell/v2/iqshell/common/data" type Redo interface { - // ShouldRedo // @Description: 是否需要重新做 // @param work 工作信息 diff --git a/iqshell/common/flow/work_creator_items.go b/iqshell/common/flow/work_creator_items.go index 93bc700b..3a6bd225 100644 --- a/iqshell/common/flow/work_creator_items.go +++ b/iqshell/common/flow/work_creator_items.go @@ -2,6 +2,7 @@ package flow import ( "fmt" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" ) diff --git a/iqshell/common/flow/work_creator_json.go b/iqshell/common/flow/work_creator_json.go index e66b4be0..50279981 100644 --- a/iqshell/common/flow/work_creator_json.go +++ b/iqshell/common/flow/work_creator_json.go @@ -2,6 +2,7 @@ package flow import ( "encoding/json" + "github.com/qiniu/qshell/v2/iqshell/common/data" ) diff --git a/iqshell/common/flow/work_limit.go b/iqshell/common/flow/work_limit.go index 1ba1a979..209414e7 100644 --- a/iqshell/common/flow/work_limit.go +++ b/iqshell/common/flow/work_limit.go @@ -1,12 +1,13 @@ package flow import ( - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/limit" "math/rand" "sync" "sync/atomic" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/limit" ) type AutoLimitOption func(l *autoLimit) diff --git a/iqshell/common/flow/work_provider.go b/iqshell/common/flow/work_provider.go index 3466aec3..bf6ac803 100644 --- a/iqshell/common/flow/work_provider.go +++ b/iqshell/common/flow/work_provider.go @@ -1,10 +1,11 @@ package flow import ( + "os" + "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "os" ) const UnknownWorkCount = int64(-1) diff --git a/iqshell/common/flow/work_provider_array.go b/iqshell/common/flow/work_provider_array.go index edf4eac4..38e41d44 100644 --- a/iqshell/common/flow/work_provider_array.go +++ b/iqshell/common/flow/work_provider_array.go @@ -1,9 +1,10 @@ package flow import ( + "sync" + "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" - "sync" ) func NewArrayWorkProvider(works []Work) (WorkProvider, *data.CodeError) { diff --git a/iqshell/common/flow/work_provider_chan.go b/iqshell/common/flow/work_provider_chan.go index 95b66d48..1261b6c8 100644 --- a/iqshell/common/flow/work_provider_chan.go +++ b/iqshell/common/flow/work_provider_chan.go @@ -2,6 +2,7 @@ package flow import ( "fmt" + "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" ) diff --git a/iqshell/common/flow/work_provider_file.go b/iqshell/common/flow/work_provider_file.go index 12f00b13..97e17724 100644 --- a/iqshell/common/flow/work_provider_file.go +++ b/iqshell/common/flow/work_provider_file.go @@ -1,9 +1,10 @@ package flow import ( + "os" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "os" ) func NewFileWorkProvider(filePath string, creator WorkCreator) (WorkProvider, *data.CodeError) { diff --git a/iqshell/common/flow/worker.go b/iqshell/common/flow/worker.go index a828a803..ea746099 100644 --- a/iqshell/common/flow/worker.go +++ b/iqshell/common/flow/worker.go @@ -6,7 +6,6 @@ import ( ) type Worker interface { - // DoWork 处理工作 // @Description: recordList 长度需和 workInfos 长度想等 // @param workInfos 工作列表 diff --git a/iqshell/common/limit/block.go b/iqshell/common/limit/block.go index 8e0f1344..f168628c 100644 --- a/iqshell/common/limit/block.go +++ b/iqshell/common/limit/block.go @@ -1,9 +1,10 @@ package limit import ( - "github.com/qiniu/qshell/v2/iqshell/common/data" "sync" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) // BlockLimit 并发 + 限流 diff --git a/iqshell/common/locker/locker.go b/iqshell/common/locker/locker.go index 47b73ff6..e12a31fc 100644 --- a/iqshell/common/locker/locker.go +++ b/iqshell/common/locker/locker.go @@ -2,17 +2,20 @@ package locker import ( "fmt" + "os" + "path/filepath" + "sync" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "os" - "path/filepath" - "sync" ) -var lockerPath string -var locker sync.Mutex +var ( + lockerPath string + locker sync.Mutex +) func SetLockerPath(path string) { if len(path) == 0 { diff --git a/iqshell/common/log/console.go b/iqshell/common/log/console.go index 66f156e2..d73fc54f 100644 --- a/iqshell/common/log/console.go +++ b/iqshell/common/log/console.go @@ -3,10 +3,11 @@ package log import ( "encoding/json" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "strings" "time" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/astaxie/beego/logs" ) @@ -38,7 +39,7 @@ var colors = []brush{ // consoleWriter implements LoggerInterface and writes messages to terminal. type consoleWriter struct { Level int `json:"level"` - Colorful bool `json:"color"` //this filed is useful only when system's terminal supports color + Colorful bool `json:"color"` // this filed is useful only when system's terminal supports color } // NewConsole create ConsoleWriter returning as LoggerInterface. diff --git a/iqshell/common/log/data.go b/iqshell/common/log/data.go index 878d607e..00e09847 100644 --- a/iqshell/common/log/data.go +++ b/iqshell/common/log/data.go @@ -2,6 +2,7 @@ package log import ( "encoding/json" + "github.com/astaxie/beego/logs" ) diff --git a/iqshell/common/log/log.go b/iqshell/common/log/log.go index ed05bd5b..861ca99c 100644 --- a/iqshell/common/log/log.go +++ b/iqshell/common/log/log.go @@ -2,13 +2,18 @@ package log import ( "fmt" + "github.com/astaxie/beego/logs" ) var progressLog *logs.BeeLogger func Debug(a ...interface{}) { - DebugF(fmt.Sprint(a...)) + if progressLog != nil { + progressLog.Debug(fmt.Sprint(a...)) + } else { + fmt.Println(a...) + } } func DebugF(format string, v ...interface{}) { @@ -21,7 +26,11 @@ func DebugF(format string, v ...interface{}) { } func Info(a ...interface{}) { - InfoF(fmt.Sprint(a...)) + if progressLog != nil { + progressLog.Info(fmt.Sprint(a...)) + } else { + fmt.Println(a...) + } } func InfoF(format string, v ...interface{}) { @@ -34,7 +43,11 @@ func InfoF(format string, v ...interface{}) { } func Warning(a ...interface{}) { - WarningF(fmt.Sprint(a...)) + if progressLog != nil { + progressLog.Warn(fmt.Sprint(a...)) + } else { + fmt.Println(a...) + } } func WarningF(format string, v ...interface{}) { @@ -47,7 +60,11 @@ func WarningF(format string, v ...interface{}) { } func Error(a ...interface{}) { - ErrorF(fmt.Sprint(a...)) + if progressLog != nil { + progressLog.Error(fmt.Sprint(a...)) + } else { + fmt.Println(a...) + } } func ErrorF(format string, v ...interface{}) { @@ -60,7 +77,11 @@ func ErrorF(format string, v ...interface{}) { } func Alert(a ...interface{}) { - AlertF(fmt.Sprint(a...)) + if progressLog != nil { + progressLog.Alert(fmt.Sprint(a...)) + } else { + fmt.Println(a...) + } } func AlertF(format string, v ...interface{}) { @@ -70,5 +91,4 @@ func AlertF(format string, v ...interface{}) { fmt.Printf(format, v...) fmt.Println("") } - } diff --git a/iqshell/common/progress/printer.go b/iqshell/common/progress/printer.go index e166470e..d1ef0834 100644 --- a/iqshell/common/progress/printer.go +++ b/iqshell/common/progress/printer.go @@ -2,9 +2,10 @@ package progress import ( "fmt" - "github.com/schollz/progressbar/v3" "sync" "time" + + "github.com/schollz/progressbar/v3" ) const ( diff --git a/iqshell/common/provider/provider.go b/iqshell/common/provider/provider.go index 64ef15f3..4944ac1b 100644 --- a/iqshell/common/provider/provider.go +++ b/iqshell/common/provider/provider.go @@ -1,9 +1,10 @@ package provider import ( - "github.com/qiniu/qshell/v2/iqshell/common/data" "sync" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) type Provider interface { diff --git a/iqshell/common/recorder/db.go b/iqshell/common/recorder/db.go index fcc09394..7f832248 100644 --- a/iqshell/common/recorder/db.go +++ b/iqshell/common/recorder/db.go @@ -1,13 +1,14 @@ package recorder import ( + "os" + "path/filepath" + "sync" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" - "os" - "path/filepath" - "sync" ) type dbRecorder struct { @@ -15,8 +16,10 @@ type dbRecorder struct { db *leveldb.DB } -var dbMap map[string]*dbRecorder -var dbMapLock sync.Mutex +var ( + dbMap map[string]*dbRecorder + dbMapLock sync.Mutex +) func CreateDBRecorder(filePath string) (Recorder, *data.CodeError) { dbMapLock.Lock() diff --git a/iqshell/common/recorder/recorder.go b/iqshell/common/recorder/recorder.go index d5aafbd7..6f096918 100644 --- a/iqshell/common/recorder/recorder.go +++ b/iqshell/common/recorder/recorder.go @@ -3,7 +3,6 @@ package recorder import "github.com/qiniu/qshell/v2/iqshell/common/data" type Recorder interface { - // Get 获取记录 Get(key string) (value string, err *data.CodeError) diff --git a/iqshell/common/scanner/scanner.go b/iqshell/common/scanner/scanner.go index 025e16b4..3f9c5100 100644 --- a/iqshell/common/scanner/scanner.go +++ b/iqshell/common/scanner/scanner.go @@ -2,10 +2,11 @@ package scanner import ( "bufio" + "os" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "os" ) type Info struct { diff --git a/iqshell/common/synchronized/sync.go b/iqshell/common/synchronized/sync.go index 71ba2c53..587fc013 100644 --- a/iqshell/common/synchronized/sync.go +++ b/iqshell/common/synchronized/sync.go @@ -1,8 +1,9 @@ package synchronized import ( - "github.com/qiniu/qshell/v2/iqshell/common/data" "sync" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) type Locker interface { diff --git a/iqshell/common/utils/cmd.go b/iqshell/common/utils/cmd.go index 37345f18..79e15128 100644 --- a/iqshell/common/utils/cmd.go +++ b/iqshell/common/utils/cmd.go @@ -2,9 +2,10 @@ package utils import ( "bytes" - "github.com/qiniu/qshell/v2/iqshell/common/data" "os" "os/exec" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) func IsCmdExist(cmd string) bool { diff --git a/iqshell/common/utils/commits.go b/iqshell/common/utils/commits.go index 7cc150d2..3840b7e3 100644 --- a/iqshell/common/utils/commits.go +++ b/iqshell/common/utils/commits.go @@ -13,7 +13,6 @@ func JsonDataTrimComments(data []byte) (data1 []byte) { } func trimCommentsLine(line []byte) []byte { - var newLine []byte var i, quoteCount int lastIdx := len(line) - 1 diff --git a/iqshell/common/utils/dir_cache.go b/iqshell/common/utils/dir_cache.go index 051cabf9..c6f648a2 100644 --- a/iqshell/common/utils/dir_cache.go +++ b/iqshell/common/utils/dir_cache.go @@ -3,13 +3,14 @@ package utils import ( "bufio" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "io" "os" "path/filepath" "strings" "time" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" ) @@ -19,12 +20,11 @@ import ( // @param cacheResultFile - cache result file path // @return (fileCount, retErr) - total file count and any error meets func DirCache(cacheRootPath string, cacheResultFile string) (int64, *data.CodeError) { - log.DebugF("cacheRootPath:`%s`", cacheRootPath) cacheRootPath = filepath.Join(cacheRootPath, "") log.DebugF("cacheRootPath after disposed:`%s`", cacheRootPath) - //check dir + // check dir rootPathFileInfo, statErr := os.Stat(cacheRootPath) if statErr != nil { log.ErrorF("Failed to stat path `%s`, %s", cacheRootPath, statErr) @@ -47,7 +47,7 @@ func DirCache(cacheRootPath string, cacheResultFile string) (int64, *data.CodeEr return 0, data.NewEmptyError().AppendError(mkErr) } - //create result file + // create result file cResultFh, createErr := os.Create(cacheResultFile) if createErr != nil { log.ErrorF("Failed to open cache file `%s`, %s", cacheResultFile, createErr) @@ -60,7 +60,7 @@ func DirCache(cacheRootPath string, cacheResultFile string) (int64, *data.CodeEr bWriter := bufio.NewWriter(cacheResultFh) defer bWriter.Flush() - //walk start + // walk start walkStart := time.Now() log.DebugF("Walk `%s` start from %s", cacheRootPath, walkStart.String()) @@ -68,11 +68,11 @@ func DirCache(cacheRootPath string, cacheResultFile string) (int64, *data.CodeEr var fileCount int64 = 0 filepath.Walk(cacheRootPath, func(path string, fi os.FileInfo, walkErr error) error { var retErr error - //check error + // check error if walkErr != nil { log.ErrorF("Walk through `%s` error, %s", path, walkErr) - //skip this dir + // skip this dir retErr = filepath.SkipDir } else { if fi.IsDir() { @@ -87,7 +87,7 @@ func DirCache(cacheRootPath string, cacheResultFile string) (int64, *data.CodeEr log.DebugF("cacheRootPath:`%s` path:`%s` relativePath:`%s`", cacheRootPath, path, relativePath) fsize := fi.Size() - //Unit is 100ns + // Unit is 100ns flmd := fi.ModTime().UnixNano() / 100 log.DebugF("Meet file `%s`, size: %d, modtime: %d", relativePath, fsize, flmd) diff --git a/iqshell/common/utils/etag.go b/iqshell/common/utils/etag.go index f0827ba6..cc0d685f 100644 --- a/iqshell/common/utils/etag.go +++ b/iqshell/common/utils/etag.go @@ -3,10 +3,11 @@ package utils import ( "crypto/sha1" "encoding/base64" - "github.com/qiniu/qshell/v2/iqshell/common/data" "io" "os" "strings" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) const ( @@ -111,7 +112,8 @@ func hashSha1s(sha1s [][]byte) []byte { return []byte{ 0x16, 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, - 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09} + 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09, + } case 1: var sha1Buf []byte sha1Buf = append(sha1Buf, 0x16) diff --git a/iqshell/common/utils/ip/parser.go b/iqshell/common/utils/ip/parser.go index e14ff444..d161b8ac 100644 --- a/iqshell/common/utils/ip/parser.go +++ b/iqshell/common/utils/ip/parser.go @@ -2,8 +2,7 @@ package ip import "github.com/qiniu/qshell/v2/iqshell/common/data" -type ParserResult interface { -} +type ParserResult interface{} type Parser interface { Parse(ip string) (ParserResult, *data.CodeError) diff --git a/iqshell/common/utils/ip/parser_ali.go b/iqshell/common/utils/ip/parser_ali.go index 6a080d5f..9c655e74 100644 --- a/iqshell/common/utils/ip/parser_ali.go +++ b/iqshell/common/utils/ip/parser_ali.go @@ -3,24 +3,22 @@ package ip import ( "encoding/json" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" "io" "net/http" "strings" -) -var ( - // IP信息查询接口地址 - aliIPParseUrls = []string{ - "https://ip.taobao.com/outGetIpInfo", - "https://ip.taobao.com/service/getIpInfo.php", - } + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" ) -type aliParser struct { +// IP信息查询接口地址 +var aliIPParseUrls = []string{ + "https://ip.taobao.com/outGetIpInfo", + "https://ip.taobao.com/service/getIpInfo.php", } +type aliParser struct{} + var _ Parser = (*aliParser)(nil) func NewAliIPParser() Parser { diff --git a/iqshell/common/utils/operations/base64.go b/iqshell/common/utils/operations/base64.go index f390ab46..13217506 100644 --- a/iqshell/common/utils/operations/base64.go +++ b/iqshell/common/utils/operations/base64.go @@ -2,6 +2,7 @@ package operations import ( "encoding/base64" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/common/utils/operations/func.go b/iqshell/common/utils/operations/func.go index e93e0e99..e73719d9 100644 --- a/iqshell/common/utils/operations/func.go +++ b/iqshell/common/utils/operations/func.go @@ -1,12 +1,13 @@ package operations import ( + "strconv" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "strconv" ) type FuncCallInfo struct { diff --git a/iqshell/common/utils/operations/ip.go b/iqshell/common/utils/operations/ip.go index 31e993bf..c7813f03 100644 --- a/iqshell/common/utils/operations/ip.go +++ b/iqshell/common/utils/operations/ip.go @@ -1,12 +1,13 @@ package operations import ( + "time" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils/ip" - "time" ) type IpQueryInfo struct { diff --git a/iqshell/common/utils/operations/reqid.go b/iqshell/common/utils/operations/reqid.go index 57be36ee..b35ac2ca 100644 --- a/iqshell/common/utils/operations/reqid.go +++ b/iqshell/common/utils/operations/reqid.go @@ -3,12 +3,13 @@ package operations import ( "encoding/base64" "fmt" + "strconv" + "time" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "strconv" - "time" ) type ReqIdInfo struct { diff --git a/iqshell/common/utils/operations/rpc.go b/iqshell/common/utils/operations/rpc.go index 5d32e25d..f6d576ac 100644 --- a/iqshell/common/utils/operations/rpc.go +++ b/iqshell/common/utils/operations/rpc.go @@ -2,10 +2,11 @@ package operations import ( "bufio" + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" - "os" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" diff --git a/iqshell/common/utils/operations/timestamp.go b/iqshell/common/utils/operations/timestamp.go index a6c1820b..a9075c49 100644 --- a/iqshell/common/utils/operations/timestamp.go +++ b/iqshell/common/utils/operations/timestamp.go @@ -1,12 +1,13 @@ package operations import ( + "strconv" + "time" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "strconv" - "time" ) type TimestampInfo struct { diff --git a/iqshell/common/utils/operations/token.go b/iqshell/common/utils/operations/token.go index e88cc13b..18d2d36c 100644 --- a/iqshell/common/utils/operations/token.go +++ b/iqshell/common/utils/operations/token.go @@ -4,6 +4,12 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell" @@ -11,11 +17,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "io" - "io/ioutil" - "net/http" - "os" - "strings" ) type TokenInfo struct { @@ -137,7 +138,7 @@ func CreateUploadToken(cfg *iqshell.Config, info UploadTokenInfo) { return } - //remove UTF-8 BOM + // remove UTF-8 BOM configData = bytes.TrimPrefix(configData, []byte("\xef\xbb\xbf")) putPolicy := new(storage.PutPolicy) diff --git a/iqshell/common/utils/operations/url.go b/iqshell/common/utils/operations/url.go index b7c3ad09..fd4814d5 100644 --- a/iqshell/common/utils/operations/url.go +++ b/iqshell/common/utils/operations/url.go @@ -1,11 +1,12 @@ package operations import ( + "net/url" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "net/url" ) type UrlInfo struct { diff --git a/iqshell/common/utils/operations/zip.go b/iqshell/common/utils/operations/zip.go index 4cce2e22..0d8a639f 100644 --- a/iqshell/common/utils/operations/zip.go +++ b/iqshell/common/utils/operations/zip.go @@ -1,12 +1,13 @@ package operations import ( + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "os" ) type ZipInfo struct { diff --git a/iqshell/common/utils/strings.go b/iqshell/common/utils/strings.go index a6a3bc97..fcb48dd6 100644 --- a/iqshell/common/utils/strings.go +++ b/iqshell/common/utils/strings.go @@ -34,7 +34,7 @@ func CreateRandString(num int) (rcode string) { func SplitString(line, sep string) []string { if len(sep) == 0 { - //strings.TrimSpace(sep) == "" + // strings.TrimSpace(sep) == "" return strings.Fields(line) } return strings.Split(line, sep) diff --git a/iqshell/common/utils/template.go b/iqshell/common/utils/template.go index 26fec90c..247b37ec 100644 --- a/iqshell/common/utils/template.go +++ b/iqshell/common/utils/template.go @@ -3,12 +3,13 @@ package utils import ( "bytes" "encoding/json" + "html/template" + "path" + "github.com/Masterminds/sprig/v3" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "html/template" - "path" ) type Template struct { diff --git a/iqshell/common/utils/unzip.go b/iqshell/common/utils/unzip.go index c0a2a9b3..8a3ddaa9 100644 --- a/iqshell/common/utils/unzip.go +++ b/iqshell/common/utils/unzip.go @@ -2,18 +2,19 @@ package utils import ( "archive/zip" - "github.com/qiniu/qshell/v2/iqshell/common/data" "io" "os" "path/filepath" "unicode/utf8" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" "golang.org/x/text/encoding/simplifiedchinese" ) func Gbk2Utf8(text string) (string, *data.CodeError) { - var gDecoder = simplifiedchinese.GBK.NewDecoder() + gDecoder := simplifiedchinese.GBK.NewDecoder() utf8Dst := make([]byte, len(text)*3) _, _, err := gDecoder.Transform(utf8Dst, []byte(text), true) if err != nil { @@ -39,12 +40,12 @@ func Unzip(zipFilePath string, unzipPath string) (err *data.CodeError) { zipFiles := zipReader.File - //list dir + // list dir for _, zipFile := range zipFiles { fileInfo := zipFile.FileHeader.FileInfo() fileName := zipFile.FileHeader.Name - //check charset utf8 or gbk + // check charset utf8 or gbk if !utf8.Valid([]byte(fileName)) { fileName, err = Gbk2Utf8(fileName) if err != nil { @@ -56,7 +57,7 @@ func Unzip(zipFilePath string, unzipPath string) (err *data.CodeError) { fullPath := filepath.Join(unzipPath, fileName) if fileInfo.IsDir() { log.Debug("Mkdir", fullPath) - mErr := os.MkdirAll(fullPath, 0775) + mErr := os.MkdirAll(fullPath, 0o775) if mErr != nil { err = data.NewEmptyError().AppendDescF("Mkdir error, %s", mErr) continue @@ -64,12 +65,12 @@ func Unzip(zipFilePath string, unzipPath string) (err *data.CodeError) { } } - //list file + // list file for _, zipFile := range zipFiles { fileInfo := zipFile.FileHeader.FileInfo() fileName := zipFile.FileHeader.Name - //check charset utf8 or gbk + // check charset utf8 or gbk if !utf8.Valid([]byte(fileName)) { fileName, err = Gbk2Utf8(fileName) if err != nil { @@ -80,9 +81,9 @@ func Unzip(zipFilePath string, unzipPath string) (err *data.CodeError) { fullPath := filepath.Join(unzipPath, fileName) if !fileInfo.IsDir() { - //to be compatible with pkzip(4.5) + // to be compatible with pkzip(4.5) fullPathDir := filepath.Dir(fullPath) - mErr := os.MkdirAll(fullPathDir, 0755) + mErr := os.MkdirAll(fullPathDir, 0o755) if mErr != nil { err = data.NewEmptyError().AppendDescF("Mkdir error, %v", mErr) continue diff --git a/iqshell/common/utils/utils.go b/iqshell/common/utils/utils.go index 5b1429fa..1322b90b 100644 --- a/iqshell/common/utils/utils.go +++ b/iqshell/common/utils/utils.go @@ -166,7 +166,6 @@ func GetFileLineCount(filePath string) (totalCount int64) { // :LinkId:RelPath // :LinkId func Encode(uri string) string { - size := len(uri) if size == 0 { return "" @@ -180,7 +179,6 @@ func Encode(uri string) string { } func Decode(encodedURI string) (uri string, err *data.CodeError) { - size := len(encodedURI) if size == 0 { return diff --git a/iqshell/common/version/operations/version.go b/iqshell/common/version/operations/version.go index d0741bf1..a1a84a48 100644 --- a/iqshell/common/version/operations/version.go +++ b/iqshell/common/version/operations/version.go @@ -7,8 +7,7 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/version" ) -type VersionInfo struct { -} +type VersionInfo struct{} func (info *VersionInfo) Check() *data.CodeError { return nil diff --git a/iqshell/common/workspace/interrupt.go b/iqshell/common/workspace/interrupt.go index d8cac72c..ea4301d2 100644 --- a/iqshell/common/workspace/interrupt.go +++ b/iqshell/common/workspace/interrupt.go @@ -1,12 +1,13 @@ package workspace import ( - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" "os" "os/signal" "sync" "sync/atomic" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" ) var ( diff --git a/iqshell/load.go b/iqshell/load.go index e5f431c6..5b7b6bae 100644 --- a/iqshell/load.go +++ b/iqshell/load.go @@ -92,7 +92,7 @@ func ShowDocumentIfNeeded(cfg *Config) bool { } func loadBase(cfg *Config) (shouldContinue bool) { - //set cpu count + // set cpu count runtime.GOMAXPROCS(runtime.NumCPU()) // 配置 user agent diff --git a/iqshell/storage/bucket/create.go b/iqshell/storage/bucket/create.go index 79f56a80..1b0307b9 100644 --- a/iqshell/storage/bucket/create.go +++ b/iqshell/storage/bucket/create.go @@ -3,6 +3,7 @@ package bucket import ( "context" "fmt" + "github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" diff --git a/iqshell/storage/bucket/domain.go b/iqshell/storage/bucket/domain.go index 7db3a585..300de2e0 100644 --- a/iqshell/storage/bucket/domain.go +++ b/iqshell/storage/bucket/domain.go @@ -2,17 +2,18 @@ package bucket import ( "fmt" + "strings" + "github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/config" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "strings" ) func DomainOfBucket(bucket string) (domain string, err *data.CodeError) { - //get domains of bucket + // get domains of bucket domainsOfBucket, gErr := AllDomainsOfBucket(bucket) if gErr != nil { err = data.NewEmptyError().AppendDescF("Get domains of bucket error: %v", gErr) @@ -104,7 +105,7 @@ func allDomainsOfBucket(cfg *config.Config, bucket string) ([]DomainInfo, *data. var domains []DomainInfo reqHost := workspace.GetConfig().Hosts.GetOneUc() reqURL := fmt.Sprintf("%s/v3/domains?tbl=%s", utils.Endpoint(cfg.IsUseHttps(), reqHost), bucket) - //reqURL = fmt.Sprintf("%s/domain?bucket=%s&type=all", utils.Endpoint(cfg.IsUseHttps(), reqHost), bucket) + // reqURL = fmt.Sprintf("%s/domain?bucket=%s&type=all", utils.Endpoint(cfg.IsUseHttps(), reqHost), bucket) err := bucketManager.Client.CredentialedCall(workspace.GetContext(), bucketManager.Mac, auth.TokenQiniu, &domains, "GET", reqURL, nil) if err != nil { if e, ok := err.(*storage.ErrorInfo); ok { diff --git a/iqshell/storage/bucket/internal/list/list.go b/iqshell/storage/bucket/internal/list/list.go index 128c6d6f..1592a175 100644 --- a/iqshell/storage/bucket/internal/list/list.go +++ b/iqshell/storage/bucket/internal/list/list.go @@ -3,6 +3,7 @@ package list import ( "context" "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/bucket/internal/list/list_v1.go b/iqshell/storage/bucket/internal/list/list_v1.go index db09e683..9a684a2d 100644 --- a/iqshell/storage/bucket/internal/list/list_v1.go +++ b/iqshell/storage/bucket/internal/list/list_v1.go @@ -2,9 +2,10 @@ package list import ( "context" + "strings" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" - "strings" ) func listBucketByV1(ctx context.Context, info ApiInfo, handler Handler) (hasMore bool, err *data.CodeError) { diff --git a/iqshell/storage/bucket/internal/list/list_v2.go b/iqshell/storage/bucket/internal/list/list_v2.go index ec072cb8..a65ba682 100644 --- a/iqshell/storage/bucket/internal/list/list_v2.go +++ b/iqshell/storage/bucket/internal/list/list_v2.go @@ -4,12 +4,13 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/url" + "github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/go-sdk/v7/client" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "io" - "net/url" ) type listBucketV2Ret struct { diff --git a/iqshell/storage/bucket/list.go b/iqshell/storage/bucket/list.go index 80c1a90e..73d40622 100644 --- a/iqshell/storage/bucket/list.go +++ b/iqshell/storage/bucket/list.go @@ -62,7 +62,8 @@ type ListObject = list.Item // List list 某个 bucket 所有的文件 func List(info ListApiInfo, objectHandler func(marker string, object ListObject) (shouldContinue bool, err *data.CodeError), - errorHandler func(marker string, err *data.CodeError)) { + errorHandler func(marker string, err *data.CodeError), +) { if objectHandler == nil { data.SetCmdStatus(data.StatusError) log.Error(alert.CannotEmpty("list bucket: object handler", "")) @@ -156,7 +157,7 @@ func List(info ListApiInfo, var lErr *data.CodeError = nil for !complete && (info.MaxRetry < 0 || retryCount <= info.MaxRetry) { lErr = nil - var hasMore = false + hasMore := false limit := info.V1Limit if info.OutputLimit > 0 { limit = info.OutputLimit - outputCount diff --git a/iqshell/storage/bucket/list_cache.go b/iqshell/storage/bucket/list_cache.go index f096df72..525476a1 100644 --- a/iqshell/storage/bucket/list_cache.go +++ b/iqshell/storage/bucket/list_cache.go @@ -1,9 +1,10 @@ package bucket import ( + "os" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "os" ) type cacheInfo struct { diff --git a/iqshell/storage/bucket/list_line.go b/iqshell/storage/bucket/list_line.go index b513577b..536abdd8 100644 --- a/iqshell/storage/bucket/list_line.go +++ b/iqshell/storage/bucket/list_line.go @@ -2,11 +2,12 @@ package bucket import ( "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/utils" "strconv" "strings" "sync" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/utils" ) const ( @@ -91,7 +92,7 @@ func getKeyItems(items []string) (bool, []string) { } var fields []string - var isKeys = true + isKeys := true for _, item := range items { field := listObjectField(item) if len(field) == 0 { diff --git a/iqshell/storage/bucket/operations/domain_list.go b/iqshell/storage/bucket/operations/domain_list.go index f8e7aecb..7ff8903a 100644 --- a/iqshell/storage/bucket/operations/domain_list.go +++ b/iqshell/storage/bucket/operations/domain_list.go @@ -1,12 +1,13 @@ package operations import ( + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/storage/bucket" - "os" ) type ListDomainInfo struct { diff --git a/iqshell/storage/object/batch/batch.go b/iqshell/storage/object/batch/batch.go index 738313be..d7e37326 100644 --- a/iqshell/storage/object/batch/batch.go +++ b/iqshell/storage/object/batch/batch.go @@ -170,7 +170,6 @@ func (h *handler) Start() { workerBuilder. WorkerProvider(flow.NewWorkerProvider(func() (flow.Worker, *data.CodeError) { return flow.NewWorker(func(workInfoList []*flow.WorkInfo) ([]*flow.WorkRecord, *data.CodeError) { - recordList := make([]*flow.WorkRecord, 0, len(workInfoList)) operationBucket := "" operationStringList := make([]string, 0, len(workInfoList)) diff --git a/iqshell/storage/object/batch/data.go b/iqshell/storage/object/batch/data.go index cd43dc45..733efc69 100644 --- a/iqshell/storage/object/batch/data.go +++ b/iqshell/storage/object/batch/data.go @@ -8,9 +8,7 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/flow" ) -var ( - defaultOperationCountPerRequest = 250 -) +var defaultOperationCountPerRequest = 250 // OperationCondition // 参考链接:https://github.com/qbox/product/blob/eb21b8c26f20e967fa51b210d267c5a4d5ca2af7/kodo/rs.md#delete-%E5%88%A0%E9%99%A4%E8%B5%84%E6%BA%90 diff --git a/iqshell/storage/object/batch/metric.go b/iqshell/storage/object/batch/metric.go index d7e52517..bfe8bf8e 100644 --- a/iqshell/storage/object/batch/metric.go +++ b/iqshell/storage/object/batch/metric.go @@ -1,9 +1,10 @@ package batch import ( - "github.com/qiniu/qshell/v2/iqshell/common/log" "sync" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/log" ) type Metric struct { diff --git a/iqshell/storage/object/copy.go b/iqshell/storage/object/copy.go index 595e311c..d8d0a543 100644 --- a/iqshell/storage/object/copy.go +++ b/iqshell/storage/object/copy.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/delete.go b/iqshell/storage/object/delete.go index 6f0e9433..0fc7ceca 100644 --- a/iqshell/storage/object/delete.go +++ b/iqshell/storage/object/delete.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/download/downloader.go b/iqshell/storage/object/download/downloader.go index 07c4dedd..0dbcd030 100644 --- a/iqshell/storage/object/download/downloader.go +++ b/iqshell/storage/object/download/downloader.go @@ -334,7 +334,7 @@ func downloadTempFileWithDownloader(dl downloader, fInfo *fileInfo, info *Downlo var tempFileHandle *os.File isExist, _ := utils.ExistFile(fInfo.tempFile) if isExist { - tempFileHandle, fErr = os.OpenFile(fInfo.tempFile, os.O_APPEND|os.O_WRONLY, 0655) + tempFileHandle, fErr = os.OpenFile(fInfo.tempFile, os.O_APPEND|os.O_WRONLY, 0o655) log.DebugF("download %s => %s from:%d", downloadUrl, fInfo.toFile, info.RangeFromBytes) } else { tempFileHandle, fErr = os.Create(fInfo.tempFile) @@ -397,7 +397,7 @@ func createDownloader(info *DownloadActionInfo) (downloader, *data.CodeError) { } func utf82GBK(text string) (string, *data.CodeError) { - var gbkEncoder = simplifiedchinese.GBK.NewEncoder() + gbkEncoder := simplifiedchinese.GBK.NewEncoder() d, err := gbkEncoder.String(text) return d, data.ConvertError(err) } diff --git a/iqshell/storage/object/download/downloader_file.go b/iqshell/storage/object/download/downloader_file.go index 30bacbba..f171d685 100644 --- a/iqshell/storage/object/download/downloader_file.go +++ b/iqshell/storage/object/download/downloader_file.go @@ -29,8 +29,7 @@ type DownloadApiInfo struct { Progress progress.Progress } -type downloaderFile struct { -} +type downloaderFile struct{} func (d *downloaderFile) Download(info *DownloadApiInfo) (response *http.Response, err *data.CodeError) { for times := 0; times < 2; times++ { @@ -88,7 +87,7 @@ func (d *downloaderFile) download(info *DownloadApiInfo) (response *http.Respons } response, rErr := client.DefaultStorageClient().DoRequest(workspace.GetContext(), "GET", info.downloadUrl, headers) if info.CheckHash && len(info.FileHash) != 0 && response != nil && response.Header != nil { - etag := fmt.Sprintf(response.Header.Get("Etag")) + etag := response.Header.Get("Etag") etag = utils.ParseEtag(etag) if len(etag) > 0 && etag != info.FileHash { return nil, data.NewEmptyError().AppendDescF("file has change, hash before:%s now:%s", info.FileHash, etag) diff --git a/iqshell/storage/object/download/file_info.go b/iqshell/storage/object/download/file_info.go index 11105224..1c42bec6 100644 --- a/iqshell/storage/object/download/file_info.go +++ b/iqshell/storage/object/download/file_info.go @@ -2,10 +2,11 @@ package download import ( "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" "os" "path/filepath" "strings" + + "github.com/qiniu/qshell/v2/iqshell/common/data" ) type fileInfo struct { @@ -58,7 +59,7 @@ func (d *fileInfo) prepare() *data.CodeError { d.fileDir = filepath.Dir(d.toAbsFile) d.tempFile = fmt.Sprintf("%s.tmp", d.toAbsFile) - err = os.MkdirAll(d.fileDir, 0775) + err = os.MkdirAll(d.fileDir, 0o775) if err != nil { return data.NewEmptyError().AppendDesc("MkdirAll failed for " + d.fileDir + " error:" + err.Error()) } diff --git a/iqshell/storage/object/download/operations/batch.go b/iqshell/storage/object/download/operations/batch.go index 10d802b6..5e7c5ee6 100644 --- a/iqshell/storage/object/download/operations/batch.go +++ b/iqshell/storage/object/download/operations/batch.go @@ -298,11 +298,11 @@ func BatchDownload(cfg *iqshell.Config, info BatchDownloadInfo) { ShouldSkip(func(workInfo *flow.WorkInfo) (skip bool, cause *data.CodeError) { apiInfo, _ := workInfo.Work.(*download.DownloadActionInfo) if filterPrefix(apiInfo.Key) { - //log.InfoF("Download Skip because key prefix doesn't match, [%s:%s]", apiInfo.Bucket, apiInfo.Key) + // log.InfoF("Download Skip because key prefix doesn't match, [%s:%s]", apiInfo.Bucket, apiInfo.Key) return true, data.NewEmptyError().AppendDescF("[%s:%s], prefix filter not match", apiInfo.Bucket, apiInfo.Key) } if filterSuffixes(apiInfo.Key) { - //log.InfoF("Download Skip because key suffix doesn't match, [%s:%s]", apiInfo.Bucket, apiInfo.Key) + // log.InfoF("Download Skip because key suffix doesn't match, [%s:%s]", apiInfo.Bucket, apiInfo.Key) return true, data.NewEmptyError().AppendDescF("[%s:%s], suffix filter not match", apiInfo.Bucket, apiInfo.Key) } return false, nil diff --git a/iqshell/storage/object/download/operations/batch_data.go b/iqshell/storage/object/download/operations/batch_data.go index c9683625..86be9f52 100644 --- a/iqshell/storage/object/download/operations/batch_data.go +++ b/iqshell/storage/object/download/operations/batch_data.go @@ -2,6 +2,7 @@ package operations import ( "fmt" + "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/utils" @@ -24,7 +25,7 @@ type DownloadCfg struct { SliceSize int64 `json:"slice_size"` SliceConcurrentCount int `json:"slice_concurrent_count"` - //down from cdn + // down from cdn Referer string `json:"referer,omitempty"` CdnDomain string `json:"cdn_domain,omitempty"` // 废弃 Domain string `json:"domain,omitempty"` diff --git a/iqshell/storage/object/download/operations/download_host.go b/iqshell/storage/object/download/operations/download_host.go index 390784c9..1b477227 100644 --- a/iqshell/storage/object/download/operations/download_host.go +++ b/iqshell/storage/object/download/operations/download_host.go @@ -33,7 +33,6 @@ func getDownloadHosts(cfg *config.Config, downloadCfg *DownloadCfg) []*host.Host } func defaultDownloadHosts(cfg *config.Config, downloadCfg *DownloadCfg) []*host.Host { - hosts := make([]*host.Host, 0) // 1. 从 download 配置获取 diff --git a/iqshell/storage/object/download/operations/utils.go b/iqshell/storage/object/download/operations/utils.go index d20f7484..7a47e0fe 100644 --- a/iqshell/storage/object/download/operations/utils.go +++ b/iqshell/storage/object/download/operations/utils.go @@ -1,11 +1,12 @@ package operations import ( + "os" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell/common/config" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "os" - "path/filepath" ) func downloadCachePath(cfg *config.Config, downloadCfg *DownloadCfg) string { diff --git a/iqshell/storage/object/download/operations/work_provider.go b/iqshell/storage/object/download/operations/work_provider.go index bde96895..c40989e2 100644 --- a/iqshell/storage/object/download/operations/work_provider.go +++ b/iqshell/storage/object/download/operations/work_provider.go @@ -93,7 +93,6 @@ func (w *workProvider) getWorkInfoFromFile() { } return info, nil })) - if err != nil { log.ErrorF("download create work provider error:%v", err) close(w.downloadItemChan) diff --git a/iqshell/storage/object/download/slice_downloader.go b/iqshell/storage/object/download/slice_downloader.go index 6eae2d24..af12d0d5 100644 --- a/iqshell/storage/object/download/slice_downloader.go +++ b/iqshell/storage/object/download/slice_downloader.go @@ -3,16 +3,17 @@ package download import ( "errors" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" - "github.com/qiniu/qshell/v2/iqshell/common/utils" - "github.com/qiniu/qshell/v2/iqshell/common/workspace" "io" "net/http" "os" "path/filepath" "sync" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" + "github.com/qiniu/qshell/v2/iqshell/common/utils" + "github.com/qiniu/qshell/v2/iqshell/common/workspace" ) type slice struct { @@ -114,7 +115,6 @@ func (s *sliceDownloader) initDownloadStatus(info *DownloadApiInfo) *data.CodeEr // 并发下载 func (s *sliceDownloader) download(info *DownloadApiInfo) (response *http.Response, err *data.CodeError) { - from := s.currentReadSliceIndex * s.SliceSize index := s.currentReadSliceIndex go func() { @@ -233,7 +233,6 @@ func (s *sliceDownloader) downloadSlice(info *DownloadApiInfo, sl slice) *data.C } func (s *sliceDownloader) Read(p []byte) (int, error) { - if s.getDownloadError() != nil { return 0, s.downloadError } diff --git a/iqshell/storage/object/fetch.go b/iqshell/storage/object/fetch.go index bc8ae040..d561a330 100644 --- a/iqshell/storage/object/fetch.go +++ b/iqshell/storage/object/fetch.go @@ -3,6 +3,8 @@ package object import ( "context" "fmt" + "strings" + "github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" @@ -11,7 +13,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/workspace" "github.com/qiniu/qshell/v2/iqshell/storage/bucket" - "strings" ) type FetchApiInfo struct { @@ -38,7 +39,6 @@ func (a *FetchResult) IsValid() bool { } func Fetch(info FetchApiInfo) (*FetchResult, *data.CodeError) { - if len(info.Bucket) == 0 { return nil, alert.CannotEmptyError("bucket", "") } diff --git a/iqshell/storage/object/m3u8/delete.go b/iqshell/storage/object/m3u8/delete.go index fcbfdd6f..8d6f328c 100644 --- a/iqshell/storage/object/m3u8/delete.go +++ b/iqshell/storage/object/m3u8/delete.go @@ -16,7 +16,6 @@ func Delete(info DeleteApiInfo) ([]*batch.OperationResult, *data.CodeError) { Bucket: info.Bucket, Key: info.Key, }) - if err != nil { return nil, data.NewEmptyError().AppendDesc("Get m3u8 file list error:" + err.Error()) } diff --git a/iqshell/storage/object/m3u8/m3u8_test.go b/iqshell/storage/object/m3u8/m3u8_test.go index e9fd90bf..9ed3f57c 100644 --- a/iqshell/storage/object/m3u8/m3u8_test.go +++ b/iqshell/storage/object/m3u8/m3u8_test.go @@ -104,7 +104,8 @@ func TestReplaceTsWithNewDomain(t *testing.T) { } func checkReplaceTsWithNewDomain(t *testing.T, line, newDomain string, - removeSparePreSlash bool, expected string) { + removeSparePreSlash bool, expected string, +) { newLine := replaceTsNewDomain(line, newDomain, removeSparePreSlash) assert.Equal(t, expected, newLine) } diff --git a/iqshell/storage/object/m3u8/operations/delete.go b/iqshell/storage/object/m3u8/operations/delete.go index 75ca149c..4a515aa0 100644 --- a/iqshell/storage/object/m3u8/operations/delete.go +++ b/iqshell/storage/object/m3u8/operations/delete.go @@ -1,12 +1,13 @@ package operations import ( + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/storage/object/m3u8" - "os" ) type DeleteInfo m3u8.DeleteApiInfo diff --git a/iqshell/storage/object/m3u8/operations/replace.go b/iqshell/storage/object/m3u8/operations/replace.go index 5ae7f390..a0c4b157 100644 --- a/iqshell/storage/object/m3u8/operations/replace.go +++ b/iqshell/storage/object/m3u8/operations/replace.go @@ -1,12 +1,13 @@ package operations import ( + "os" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/storage/object/m3u8" - "os" ) type ReplaceDomainInfo m3u8.ReplaceDomainApiInfo diff --git a/iqshell/storage/object/m3u8/replace.go b/iqshell/storage/object/m3u8/replace.go index 75438478..0db679be 100644 --- a/iqshell/storage/object/m3u8/replace.go +++ b/iqshell/storage/object/m3u8/replace.go @@ -4,16 +4,17 @@ import ( "bufio" "bytes" "fmt" - "github.com/qiniu/go-sdk/v7/storage" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" - "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "github.com/qiniu/qshell/v2/iqshell/storage/object/download" "io/ioutil" "net/http" "net/url" "strings" "time" + + "github.com/qiniu/go-sdk/v7/storage" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" + "github.com/qiniu/qshell/v2/iqshell/common/workspace" + "github.com/qiniu/qshell/v2/iqshell/storage/object/download" ) // replace and upload @@ -30,7 +31,7 @@ func ReplaceDomain(info ReplaceDomainApiInfo) *data.CodeError { Key: info.Key, }) - //create download link + // create download link if urlResult, e := download.PublicUrlToPrivate(download.PublicUrlToPrivateApiInfo{ PublicUrl: dnLink, Deadline: time.Now().Add(time.Second * 3600).Unix(), @@ -40,7 +41,7 @@ func ReplaceDomain(info ReplaceDomainApiInfo) *data.CodeError { dnLink = urlResult.Url } - //get m3u8 file content + // get m3u8 file content m3u8Req, reqErr := http.NewRequest("GET", dnLink, nil) if reqErr != nil { return data.NewEmptyError().AppendDescF("new request for url %s error, %s", dnLink, reqErr) @@ -61,7 +62,7 @@ func ReplaceDomain(info ReplaceDomainApiInfo) *data.CodeError { return data.NewEmptyError().AppendDescF("read m3u8 file content error, %s", readErr.Error()) } - //check content + // check content if !strings.HasPrefix(string(m3u8Bytes), "#EXTM3U") { return data.NewEmptyError().AppendDesc("invalid m3u8 file") } @@ -76,7 +77,7 @@ func ReplaceDomain(info ReplaceDomainApiInfo) *data.CodeError { newM3u8Lines = append(newM3u8Lines, newLine) } - //join and upload + // join and upload newM3u8Data := []byte(strings.Join(newM3u8Lines, "\n")) putPolicy := storage.PutPolicy{ Scope: fmt.Sprintf("%s:%s", info.Bucket, info.Key), diff --git a/iqshell/storage/object/m3u8/slices.go b/iqshell/storage/object/m3u8/slices.go index 73ee65bb..1fb9e662 100644 --- a/iqshell/storage/object/m3u8/slices.go +++ b/iqshell/storage/object/m3u8/slices.go @@ -4,15 +4,16 @@ import ( "bufio" "bytes" "fmt" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" - "github.com/qiniu/qshell/v2/iqshell/storage/bucket" - "github.com/qiniu/qshell/v2/iqshell/storage/object/download" "io/ioutil" "net/http" "net/url" "strings" "time" + + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" + "github.com/qiniu/qshell/v2/iqshell/storage/bucket" + "github.com/qiniu/qshell/v2/iqshell/storage/object/download" ) type Slice struct { @@ -41,7 +42,7 @@ func Slices(info SliceListApiInfo) ([]Slice, *data.CodeError) { dnLink = urlResult.Url } - //get m3u8 file content + // get m3u8 file content m3u8Req, reqErr := http.NewRequest("GET", dnLink, nil) if reqErr != nil { return nil, data.NewEmptyError().AppendDescF("new request for url %s error, %s", dnLink, reqErr) @@ -62,7 +63,7 @@ func Slices(info SliceListApiInfo) ([]Slice, *data.CodeError) { return nil, data.NewEmptyError().AppendDescF("read m3u8 file content error, %s", readErr.Error()) } - //check content + // check content if !strings.HasPrefix(string(m3u8Bytes), "#EXTM3U") { return nil, data.NewEmptyError().AppendDesc("invalid m3u8 file") } @@ -126,7 +127,7 @@ func downloadLink(info downloadLinkApiInfo) (dnLink string, err *data.CodeError) } } - //get first + // get first if domain == "" && len(bucketDomains) > 0 { domain = bucketDomains[0].Domain } diff --git a/iqshell/storage/object/match.go b/iqshell/storage/object/match.go index 80282422..1753b77f 100644 --- a/iqshell/storage/object/match.go +++ b/iqshell/storage/object/match.go @@ -2,11 +2,12 @@ package object import ( "fmt" + "os" + "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/flow" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/utils" - "os" ) const ( diff --git a/iqshell/storage/object/mime_change.go b/iqshell/storage/object/mime_change.go index b2492b7f..0250459c 100644 --- a/iqshell/storage/object/mime_change.go +++ b/iqshell/storage/object/mime_change.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/move.go b/iqshell/storage/object/move.go index 750e6cdd..a1c4b50b 100644 --- a/iqshell/storage/object/move.go +++ b/iqshell/storage/object/move.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/operations/delete.go b/iqshell/storage/object/operations/delete.go index 519f10de..479da7b2 100644 --- a/iqshell/storage/object/operations/delete.go +++ b/iqshell/storage/object/operations/delete.go @@ -2,6 +2,9 @@ package operations import ( "fmt" + "path/filepath" + "strconv" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -12,8 +15,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/storage/bucket" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" - "strconv" ) type DeleteInfo struct { diff --git a/iqshell/storage/object/operations/fetch.go b/iqshell/storage/object/operations/fetch.go index eff313e2..33daf6bd 100644 --- a/iqshell/storage/object/operations/fetch.go +++ b/iqshell/storage/object/operations/fetch.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -12,7 +14,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/workspace" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) type FetchInfo object.FetchApiInfo @@ -171,7 +172,6 @@ func BatchFetch(cfg *iqshell.Config, info BatchFetchInfo) { exporter.Fail().ExportF("%s%s%v", work.Data, flow.ErrorSeparate, err) log.InfoF("Skip line:%s because:%v", work.Data, err) } - }). OnWorkSuccess(func(workInfo *flow.WorkInfo, result flow.Result) { metric.AddCurrentCount(1) diff --git a/iqshell/storage/object/operations/fetch_async.go b/iqshell/storage/object/operations/fetch_async.go index 1fd9f544..02c60c14 100644 --- a/iqshell/storage/object/operations/fetch_async.go +++ b/iqshell/storage/object/operations/fetch_async.go @@ -115,8 +115,8 @@ func BatchAsyncFetch(cfg *iqshell.Config, info BatchAsyncFetchInfo) { } func batchAsyncFetch(cfg *iqshell.Config, info BatchAsyncFetchInfo, - exporter *export.FileExporter, fetchResultChan chan<- flow.Work) { - + exporter *export.FileExporter, fetchResultChan chan<- flow.Work, +) { metric := &batch.Metric{} metric.Start() @@ -304,7 +304,8 @@ func batchAsyncFetch(cfg *iqshell.Config, info BatchAsyncFetchInfo, } func batchAsyncFetchCheck(cfg *iqshell.Config, info BatchAsyncFetchInfo, - exporter *export.FileExporter, fetchResultChan <-chan flow.Work) { + exporter *export.FileExporter, fetchResultChan <-chan flow.Work, +) { if info.DisableCheckFetchResult { log.DebugF("batch async fetch check: disable") for r := range fetchResultChan { @@ -505,8 +506,10 @@ type asyncFetchResult struct { Info *object.AsyncFetchApiResult `json:"info"` } -var _ flow.Work = (*asyncFetchResult)(nil) -var _ flow.Result = (*asyncFetchResult)(nil) +var ( + _ flow.Work = (*asyncFetchResult)(nil) + _ flow.Result = (*asyncFetchResult)(nil) +) func (f *asyncFetchResult) String() string { return fmt.Sprintf("%s => [%s:%s]", f.Url, f.Bucket, f.Key) diff --git a/iqshell/storage/object/operations/lifecycle.go b/iqshell/storage/object/operations/lifecycle.go index d8210b42..4be58c41 100644 --- a/iqshell/storage/object/operations/lifecycle.go +++ b/iqshell/storage/object/operations/lifecycle.go @@ -53,10 +53,14 @@ func ChangeLifecycle(cfg *iqshell.Config, info *ChangeLifecycleInfo) { } if result.IsSuccess() { - lifecycleValues := []int{info.ToIAAfterDays, info.ToIntelligentTieringAfterDays, info.ToArchiveIRAfterDays, info.ToArchiveAfterDays, - info.ToDeepArchiveAfterDays, info.DeleteAfterDays} - lifecycleDescs := []string{"to IA storage", "to IntelligentTiering storage", "to ARCHIVE_IR storage", "to ARCHIVE storage", - "to DEEP_ARCHIVE storage", "delete"} + lifecycleValues := []int{ + info.ToIAAfterDays, info.ToIntelligentTieringAfterDays, info.ToArchiveIRAfterDays, info.ToArchiveAfterDays, + info.ToDeepArchiveAfterDays, info.DeleteAfterDays, + } + lifecycleDescs := []string{ + "to IA storage", "to IntelligentTiering storage", "to ARCHIVE_IR storage", "to ARCHIVE storage", + "to DEEP_ARCHIVE storage", "delete", + } log.InfoF("Change lifecycle Success, [%s:%s]", info.Bucket, info.Key) for i := 0; i < len(lifecycleValues); i++ { lifecycleValue := lifecycleValues[i] diff --git a/iqshell/storage/object/operations/match.go b/iqshell/storage/object/operations/match.go index e1c33ede..21b5a50b 100644 --- a/iqshell/storage/object/operations/match.go +++ b/iqshell/storage/object/operations/match.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -13,7 +15,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/storage/bucket" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) type MatchInfo object.MatchApiInfo diff --git a/iqshell/storage/object/operations/mime_change.go b/iqshell/storage/object/operations/mime_change.go index ab1dad6e..69e6f93f 100644 --- a/iqshell/storage/object/operations/mime_change.go +++ b/iqshell/storage/object/operations/mime_change.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -11,7 +13,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) type ChangeMimeInfo object.ChangeMimeApiInfo diff --git a/iqshell/storage/object/operations/move.go b/iqshell/storage/object/operations/move.go index 5c02bcb2..cbc60ac4 100644 --- a/iqshell/storage/object/operations/move.go +++ b/iqshell/storage/object/operations/move.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -11,7 +13,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) type MoveInfo object.MoveApiInfo diff --git a/iqshell/storage/object/operations/private_url.go b/iqshell/storage/object/operations/private_url.go index 71f748e0..1ea44bb1 100644 --- a/iqshell/storage/object/operations/private_url.go +++ b/iqshell/storage/object/operations/private_url.go @@ -204,7 +204,6 @@ func BatchPrivateUrl(cfg *iqshell.Config, info BatchPrivateUrlInfo) { exporter.Fail().ExportF("%s%s%v", work.Data, flow.ErrorSeparate, err) log.DebugF("Skip line:%s because:%v", work.Data, err) } - }). OnWorkSuccess(func(work *flow.WorkInfo, result flow.Result) { metric.AddCurrentCount(1) diff --git a/iqshell/storage/object/operations/rename.go b/iqshell/storage/object/operations/rename.go index 21b198c5..6a1b6a49 100644 --- a/iqshell/storage/object/operations/rename.go +++ b/iqshell/storage/object/operations/rename.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -11,7 +13,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) // rename 实际用的还是 move diff --git a/iqshell/storage/object/operations/restore_archive.go b/iqshell/storage/object/operations/restore_archive.go index 8d00faca..ce23c887 100644 --- a/iqshell/storage/object/operations/restore_archive.go +++ b/iqshell/storage/object/operations/restore_archive.go @@ -2,6 +2,9 @@ package operations import ( "fmt" + "path/filepath" + "strconv" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -11,8 +14,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" - "strconv" ) func convertFreezeAfterDaysToInt(freezeAfterDays string) (int, *data.CodeError) { diff --git a/iqshell/storage/object/operations/share.go b/iqshell/storage/object/operations/share.go index 1767610a..60eae80f 100644 --- a/iqshell/storage/object/operations/share.go +++ b/iqshell/storage/object/operations/share.go @@ -396,9 +396,9 @@ func copyShare(cfg *iqshell.Config, info *CopyShareInfo) error { downloadPath = filepath.Join(toPath, relativePath) } if strings.HasSuffix(aws.StringValue(s3Object.Key), "/") && aws.Int64Value(s3Object.Size) == 0 { - err = os.MkdirAll(downloadPath, 0700) + err = os.MkdirAll(downloadPath, 0o700) } else { - if err = os.MkdirAll(filepath.Dir(downloadPath), 0700); err != nil { + if err = os.MkdirAll(filepath.Dir(downloadPath), 0o700); err != nil { return err } err = s3DownloadObjectToPath(s3Downloader, response.BucketId, aws.StringValue(s3Object.Key), downloadPath) @@ -421,11 +421,11 @@ func copyShare(cfg *iqshell.Config, info *CopyShareInfo) error { fromPrefix = strings.TrimSuffix(fromPrefix, "/") downloadPath = filepath.Join(downloadPath, fromPrefix[(strings.LastIndex(fromPrefix, "/")+1):]) if onlyMkDir { - if err = os.MkdirAll(downloadPath, 0700); err != nil { + if err = os.MkdirAll(downloadPath, 0o700); err != nil { return err } } else { - if err = os.MkdirAll(filepath.Dir(downloadPath), 0700); err != nil { + if err = os.MkdirAll(filepath.Dir(downloadPath), 0o700); err != nil { return err } if err = s3DownloadObjectToPath(s3Downloader, response.BucketId, fromPrefix, downloadPath); err != nil { @@ -465,7 +465,7 @@ func s3StatObject(s3Service *s3.S3, fromBucketId, key string) error { } func s3DownloadObjectToPath(downloader *s3manager.Downloader, fromBucketId, key, downloadPath string) error { - file, err := os.OpenFile(downloadPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + file, err := os.OpenFile(downloadPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) if err != nil { return err } diff --git a/iqshell/storage/object/operations/status_change.go b/iqshell/storage/object/operations/status_change.go index 0322700d..2295ce9d 100644 --- a/iqshell/storage/object/operations/status_change.go +++ b/iqshell/storage/object/operations/status_change.go @@ -2,6 +2,8 @@ package operations import ( "fmt" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" @@ -11,7 +13,6 @@ import ( "github.com/qiniu/qshell/v2/iqshell/common/utils" "github.com/qiniu/qshell/v2/iqshell/storage/object" "github.com/qiniu/qshell/v2/iqshell/storage/object/batch" - "path/filepath" ) type ForbiddenInfo struct { diff --git a/iqshell/storage/object/restore_archive.go b/iqshell/storage/object/restore_archive.go index 8ef65a83..1748192b 100644 --- a/iqshell/storage/object/restore_archive.go +++ b/iqshell/storage/object/restore_archive.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/save_as.go b/iqshell/storage/object/save_as.go index 353a7cdc..04c405c8 100644 --- a/iqshell/storage/object/save_as.go +++ b/iqshell/storage/object/save_as.go @@ -4,10 +4,11 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/base64" + "net/url" + "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "net/url" ) type SaveAsApiInfo struct { diff --git a/iqshell/storage/object/type_change.go b/iqshell/storage/object/type_change.go index 9b58a57c..b2437b54 100644 --- a/iqshell/storage/object/type_change.go +++ b/iqshell/storage/object/type_change.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/alert" "github.com/qiniu/qshell/v2/iqshell/common/data" diff --git a/iqshell/storage/object/upload/api/recorder.go b/iqshell/storage/object/upload/api/recorder.go index 7e167cce..4fab72d0 100644 --- a/iqshell/storage/object/upload/api/recorder.go +++ b/iqshell/storage/object/upload/api/recorder.go @@ -2,11 +2,12 @@ package api import ( "encoding/json" + "os" + "time" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "os" - "time" ) type ProgressRecorder struct { @@ -30,9 +31,9 @@ func NewProgressRecorder(filePath string) *ProgressRecorder { func (p *ProgressRecorder) Recover() (err *data.CodeError) { if statInfo, statErr := os.Stat(p.FilePath); statErr == nil { - //check file last modified time, if older than one week, ignore + // check file last modified time, if older than one week, ignore if statInfo.ModTime().Add(time.Hour * 24 * 5).After(time.Now()) { - //try read old progress + // try read old progress progressFh, openErr := os.Open(p.FilePath) if openErr != nil { err = data.NewEmptyError().AppendError(openErr) @@ -54,8 +55,7 @@ func (p *ProgressRecorder) Reset() { } func (p *ProgressRecorder) CheckValid(fileSize int64, lastModified int, isResumableV2 bool) { - - //check offset valid or not + // check offset valid or not if p.Offset%data.BLOCK_SIZE != 0 { log.Info("Invalid offset from progress file,", p.Offset) p.Reset() @@ -64,7 +64,7 @@ func (p *ProgressRecorder) CheckValid(fileSize int64, lastModified int, isResuma // 分片 V1 if !isResumableV2 { - //check offset and blk ctxs + // check offset and blk ctxs if p.Offset != 0 && p.BlkCtxs != nil && int(p.Offset/data.BLOCK_SIZE) != len(p.BlkCtxs) { log.Info("Invalid offset and block info") @@ -72,7 +72,7 @@ func (p *ProgressRecorder) CheckValid(fileSize int64, lastModified int, isResuma return } - //check blk ctxs, when no progress found + // check blk ctxs, when no progress found if p.Offset == 0 || p.BlkCtxs == nil { p.Reset() return @@ -98,7 +98,7 @@ func (p *ProgressRecorder) CheckValid(fileSize int64, lastModified int, isResuma } // 分片 V2 - //check offset and blk ctxs + // check offset and blk ctxs if p.Offset != 0 && p.Parts != nil && int(p.Offset/data.BLOCK_SIZE) != len(p.Parts) { log.Info("Invalid offset and block info") @@ -106,7 +106,7 @@ func (p *ProgressRecorder) CheckValid(fileSize int64, lastModified int, isResuma return } - //check blk ctxs, when no progress found + // check blk ctxs, when no progress found if p.Offset == 0 || p.Parts == nil { p.Reset() return diff --git a/iqshell/storage/object/upload/api/resume.go b/iqshell/storage/object/upload/api/resume.go index 6bb1d339..80e9897a 100644 --- a/iqshell/storage/object/upload/api/resume.go +++ b/iqshell/storage/object/upload/api/resume.go @@ -2,10 +2,11 @@ package api import ( "context" + "time" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/qiniu/qshell/v2/iqshell/common/log" - "time" ) type Resume interface { diff --git a/iqshell/storage/object/upload/api/resume_v1.go b/iqshell/storage/object/upload/api/resume_v1.go index dbbfc2ac..68fd9cec 100644 --- a/iqshell/storage/object/upload/api/resume_v1.go +++ b/iqshell/storage/object/upload/api/resume_v1.go @@ -3,6 +3,7 @@ package api import ( "bytes" "context" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" ) diff --git a/iqshell/storage/object/upload/api/resume_v2.go b/iqshell/storage/object/upload/api/resume_v2.go index 42872747..143e8590 100644 --- a/iqshell/storage/object/upload/api/resume_v2.go +++ b/iqshell/storage/object/upload/api/resume_v2.go @@ -5,10 +5,11 @@ import ( "context" "crypto/md5" "encoding/hex" + "time" + "github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/qshell/v2/iqshell/common/data" "github.com/syndtr/goleveldb/leveldb/errors" - "time" ) type resumeV2 struct { diff --git a/iqshell/storage/object/upload/conveyor.go b/iqshell/storage/object/upload/conveyor.go index d78c9891..eea2fe14 100644 --- a/iqshell/storage/object/upload/conveyor.go +++ b/iqshell/storage/object/upload/conveyor.go @@ -3,12 +3,6 @@ package upload import ( "bytes" "fmt" - "github.com/qiniu/go-sdk/v7/storage" - "github.com/qiniu/qshell/v2/iqshell/common/data" - "github.com/qiniu/qshell/v2/iqshell/common/log" - "github.com/qiniu/qshell/v2/iqshell/common/utils" - "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "github.com/qiniu/qshell/v2/iqshell/storage/object/upload/api" "io" "net/http" "os" @@ -16,6 +10,13 @@ import ( "strconv" "strings" "time" + + "github.com/qiniu/go-sdk/v7/storage" + "github.com/qiniu/qshell/v2/iqshell/common/data" + "github.com/qiniu/qshell/v2/iqshell/common/log" + "github.com/qiniu/qshell/v2/iqshell/common/utils" + "github.com/qiniu/qshell/v2/iqshell/common/workspace" + "github.com/qiniu/qshell/v2/iqshell/storage/object/upload/api" ) const ( @@ -88,7 +89,7 @@ func (c *conveyor) upload(info *ApiInfo) (ret *ApiResult, err *data.CodeError) { } // 2. 上传文件分片 - var blockSize = info.ChunkSize + blockSize := info.ChunkSize if blockSize < resumeV2MinChunkSize { blockSize = int64(data.BLOCK_SIZE) } @@ -101,8 +102,8 @@ func (c *conveyor) upload(info *ApiInfo) (ret *ApiResult, err *data.CodeError) { } } - totalBlkCnt := storage.BlockCount(info.LocalFileSize) //range get and mkblk upload - rangeStartOffset := recorder.Offset //init the range offset + totalBlkCnt := storage.BlockCount(info.LocalFileSize) // range get and mkblk upload + rangeStartOffset := recorder.Offset // init the range offset fromBlkIndex := int(rangeStartOffset / data.BLOCK_SIZE) if info.Progress != nil { @@ -141,7 +142,7 @@ func (c *conveyor) upload(info *ApiInfo) (ret *ApiResult, err *data.CodeError) { } } - //advance range offset + // advance range offset rangeStartOffset += data.BLOCK_SIZE if sErr := recorder.RecordProgress(); sErr != nil { log.WarningF("sync save record progress error:%v", sErr) @@ -159,7 +160,7 @@ func (c *conveyor) upload(info *ApiInfo) (ret *ApiResult, err *data.CodeError) { } } - //delete progress file + // delete progress file if rErr := os.Remove(progressFile); rErr != nil { log.WarningF("sync remove record progress error:%v", rErr) } @@ -168,18 +169,17 @@ func (c *conveyor) upload(info *ApiInfo) (ret *ApiResult, err *data.CodeError) { } func ProgressFileFromUrl(srcResUrl, bucket, key string) (progressFile string, err *data.CodeError) { - - //create sync id + // create sync id syncId := utils.Md5Hex(fmt.Sprintf("%s:%s:%s", srcResUrl, bucket, key)) - //local storage path + // local storage path QShellRootPath := workspace.GetWorkspace() if QShellRootPath == "" { err = data.NewEmptyError().AppendDescF("empty root path\n") return } storePath := filepath.Join(QShellRootPath, ".qshell", "sync") - if mkdirErr := os.MkdirAll(storePath, 0775); mkdirErr != nil { + if mkdirErr := os.MkdirAll(storePath, 0o775); mkdirErr != nil { err = data.NewEmptyError().AppendDescF("sync Failed to mkdir `%s` due to `%s`", storePath, mkdirErr) return } @@ -189,14 +189,14 @@ func ProgressFileFromUrl(srcResUrl, bucket, key string) (progressFile string, er } func getRange(srcResUrl string, totalSize, rangeStartOffset, rangeBlockSize int64) (buffer *bytes.Buffer, err *data.CodeError) { - //range get + // range get dReq, dReqErr := http.NewRequest("GET", srcResUrl, nil) if dReqErr != nil { err = data.NewEmptyError().AppendDescF("New request error, %s", dReqErr.Error()) return } - //set range header + // set range header rangeEndOffset := rangeStartOffset + rangeBlockSize - 1 if rangeEndOffset >= totalSize { rangeEndOffset = totalSize - 1 @@ -204,7 +204,7 @@ func getRange(srcResUrl string, totalSize, rangeStartOffset, rangeBlockSize int6 dReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", rangeStartOffset, rangeEndOffset)) - //set client properties + // set client properties client := http.DefaultClient client.Timeout = httpTimeout //client.Transport = &http.Transport{ @@ -216,7 +216,7 @@ func getRange(srcResUrl string, totalSize, rangeStartOffset, rangeBlockSize int6 return nil } - //get response + // get response dResp, dRespErr := client.Do(dReq) if dRespErr != nil { err = data.NewEmptyError().AppendDescF("Get response error, %s", dRespErr.Error()) @@ -224,29 +224,29 @@ func getRange(srcResUrl string, totalSize, rangeStartOffset, rangeBlockSize int6 } defer dResp.Body.Close() - //status error + // status error if dResp.StatusCode/100 != 2 { err = data.NewEmptyError().AppendDescF("Get resource error, %s", dResp.Status) return } - //if not support range, go back and err + // if not support range, go back and err if dResp.Header.Get("Content-Range") == "" { err = data.NewEmptyError().AppendDesc("sync Remote server not support range") return } - //parse content-range + // parse content-range contentRange := dResp.Header.Get("Content-Range") rangeSize, _ := parseContentRange(contentRange) - //check ranged block size + // check ranged block size if rangeSize != (rangeEndOffset - rangeStartOffset + 1) { err = data.NewEmptyError().AppendDesc("sync Block read error, only the last range block can has bytes less than ") return } - //read content + // read content buffer = bytes.NewBuffer(nil) cpCnt, cpErr := io.Copy(buffer, dResp.Body) if cpErr != nil || cpCnt != rangeSize { @@ -274,7 +274,6 @@ func parseContentRange(contentRange string) (rangeSize, totalSize int64) { } func getUpHost(cfg *storage.Config, ak, bucket string) (upHost string, err *data.CodeError) { - var zone *storage.Zone if cfg.Zone != nil { zone = cfg.Zone diff --git a/iqshell/storage/object/upload/operations/batch.go b/iqshell/storage/object/upload/operations/batch.go index 0779a418..1af1b9da 100644 --- a/iqshell/storage/object/upload/operations/batch.go +++ b/iqshell/storage/object/upload/operations/batch.go @@ -162,7 +162,6 @@ func BatchUpload2(cfg *iqshell.Config, info BatchUpload2Info) { } func batchUpload(info BatchUpload2Info) { - log.DebugF("upload config:%+v", info) dbPath := filepath.Join(workspace.GetJobDir(), ".ldb") log.InfoF("upload status db file path:%s", dbPath) @@ -179,7 +178,7 @@ func batchUpload(info BatchUpload2Info) { } else { info.InputFile = filepath.Join(workspace.GetJobDir(), ".cache") if _, statErr := os.Stat(info.InputFile); statErr == nil { - //file exists + // file exists needScanLocal = info.IsRescanLocal() } else { needScanLocal = true @@ -234,23 +233,23 @@ func batchUploadFlow(info BatchUpload2Info, uploadConfig UploadConfig, dbPath st 3, func(items []string) (work flow.Work, err *data.CodeError) { fileRelativePath := items[0] - //pack the upload file key + // pack the upload file key fileSize, _ := strconv.ParseInt(items[1], 10, 64) modifyTime, _ := strconv.ParseInt(items[2], 10, 64) key := fileRelativePath - //check ignore dir + // check ignore dir if uploadConfig.IsIgnoreDir() { key = filepath.Base(key) } - //check prefix + // check prefix if data.NotEmpty(uploadConfig.KeyPrefix) { key = strings.Join([]string{uploadConfig.KeyPrefix, key}, "") } - //convert \ to / under windows + // convert \ to / under windows if utils.IsWindowsOS() { key = strings.Replace(key, "\\", "/", -1) } - //check file encoding + // check file encoding if data.NotEmpty(uploadConfig.FileEncoding) && utils.IsGBKEncoding(uploadConfig.FileEncoding) { key, _ = utils.Gbk2Utf8(key) } @@ -480,8 +479,7 @@ func batchUploadFlow(info BatchUpload2Info, uploadConfig UploadConfig, dbPath st } } -type BatchUploadConfigMouldInfo struct { -} +type BatchUploadConfigMouldInfo struct{} func BatchUploadConfigMould(cfg *iqshell.Config, info BatchUploadConfigMouldInfo) { log.Alert(uploadConfigMouldJsonString) diff --git a/iqshell/storage/object/upload/operations/batch_data.go b/iqshell/storage/object/upload/operations/batch_data.go index 67f170a5..66b57f39 100644 --- a/iqshell/storage/object/upload/operations/batch_data.go +++ b/iqshell/storage/object/upload/operations/batch_data.go @@ -15,7 +15,7 @@ type UploadConfig struct { UpHost string `json:"up_host,omitempty"` BindUpIp string `json:"bind_up_ip,omitempty"` BindRsIp string `json:"bind_rs_ip,omitempty"` - BindNicIp string `json:"bind_nic_ip,omitempty"` //local network interface card config + BindNicIp string `json:"bind_nic_ip,omitempty"` // local network interface card config SrcDir string `json:"src_dir,omitempty"` FileList string `json:"file_list,omitempty"` @@ -120,7 +120,7 @@ func DefaultUploadConfig() UploadConfig { SkipSuffixes: "", FileEncoding: "", Bucket: "", - ResumableAPIV2: false, + ResumableAPIV2: true, ResumableAPIV2PartSize: 4 * 1024 * 1024, PutThreshold: 8 * 1024 * 1024, KeyPrefix: "", @@ -235,9 +235,8 @@ func (up *UploadConfig) Check() *data.CodeError { } func (up *UploadConfig) HitByPathPrefixes(localFileRelativePath string) (hit bool, pathPrefix string) { - if len(up.SkipPathPrefixes) > 0 { - //unpack skip prefix + // unpack skip prefix pathPrefixes := strings.Split(up.SkipPathPrefixes, ",") for _, prefix := range pathPrefixes { if strings.TrimSpace(prefix) == "" { @@ -256,7 +255,7 @@ func (up *UploadConfig) HitByPathPrefixes(localFileRelativePath string) (hit boo func (up *UploadConfig) HitByFilePrefixes(localFileRelativePath string) (hit bool, filePrefix string) { if len(up.SkipFilePrefixes) > 0 { - //unpack skip prefix + // unpack skip prefix filePrefixes := strings.Split(up.SkipFilePrefixes, ",") for _, prefix := range filePrefixes { if strings.TrimSpace(prefix) == "" { @@ -276,7 +275,7 @@ func (up *UploadConfig) HitByFilePrefixes(localFileRelativePath string) (hit boo func (up *UploadConfig) HitByFixesString(localFileRelativePath string) (hit bool, hitFixedStr string) { if len(up.SkipFixedStrings) > 0 { - //unpack fixed strings + // unpack fixed strings fixedStrings := strings.Split(up.SkipFixedStrings, ",") for _, fixedStr := range fixedStrings { if strings.TrimSpace(fixedStr) == "" { @@ -291,7 +290,6 @@ func (up *UploadConfig) HitByFixesString(localFileRelativePath string) (hit bool } } return - } func (up *UploadConfig) HitBySuffixes(localFileRelativePath string) (hit bool, hitSuffix string) { diff --git a/iqshell/storage/object/upload/operations/config_mould.go b/iqshell/storage/object/upload/operations/config_mould.go index 222bd9e4..08b9387c 100644 --- a/iqshell/storage/object/upload/operations/config_mould.go +++ b/iqshell/storage/object/upload/operations/config_mould.go @@ -15,7 +15,7 @@ const uploadConfigMouldJsonString = `{ "skip_suffixes": "", "file_encoding": "", "bucket": "", - "resumable_api_v2": "false", + "resumable_api_v2": "true", "resumable_api_v2_part_size": 1048576, "put_threshold": 16777216, "key_prefix": "", diff --git a/iqshell/storage/object/upload/operations/upload.go b/iqshell/storage/object/upload/operations/upload.go index dea5fe0e..4c172274 100644 --- a/iqshell/storage/object/upload/operations/upload.go +++ b/iqshell/storage/object/upload/operations/upload.go @@ -122,7 +122,7 @@ func uploadFile(info *UploadInfo) (res *upload.ApiResult, err *data.CodeError) { } else { log.AlertF("Upload File success %s => [%s:%s] duration:%.2fs Speed:%s", info.FilePath, info.ToBucket, info.SaveKey, duration, speed) - //delete on success + // delete on success if info.DeleteOnSuccess { deleteErr := os.Remove(info.FilePath) if deleteErr != nil { diff --git a/iqshell/storage/object/upload/operations/utils.go b/iqshell/storage/object/upload/operations/utils.go index 407964d4..db625f46 100644 --- a/iqshell/storage/object/upload/operations/utils.go +++ b/iqshell/storage/object/upload/operations/utils.go @@ -1,11 +1,12 @@ package operations import ( + "os" + "path/filepath" + "github.com/qiniu/qshell/v2/iqshell/common/config" "github.com/qiniu/qshell/v2/iqshell/common/log" "github.com/qiniu/qshell/v2/iqshell/common/workspace" - "os" - "path/filepath" ) func uploadCachePath(cfg *config.Config, uploadCfg *UploadConfig) string { diff --git a/iqshell/storage/servers/all_buckets.go b/iqshell/storage/servers/all_buckets.go index 7e33d145..c9a528c1 100644 --- a/iqshell/storage/servers/all_buckets.go +++ b/iqshell/storage/servers/all_buckets.go @@ -17,8 +17,7 @@ type UserInfo struct { Perm *data.Int `json:"perm"` } -type BucketQuota struct { -} +type BucketQuota struct{} type BucketInfo struct { Name *data.String `json:"name"` @@ -27,14 +26,14 @@ type BucketInfo struct { StorageSize *data.Int64 `json:"storage_size"` Region *data.String `json:"region"` - //CTime *data.String `json:"ctime"` - //Global *data.Bool `json:"global"` - //Perm *data.Int `json:"perm"` - //ShareUsers []*UserInfo `json:"share_users"` - //Versioning *data.Bool `json:"versioning"` - //AllowNullKey *data.Bool `json:"allow_nullkey"` - //EncryptionEnabled *data.Bool `json:"encryption_enabled"` - //NotAllowAccessByTbl *data.Bool `json:"not_allow_access_by_tbl"` + // CTime *data.String `json:"ctime"` + // Global *data.Bool `json:"global"` + // Perm *data.Int `json:"perm"` + // ShareUsers []*UserInfo `json:"share_users"` + // Versioning *data.Bool `json:"versioning"` + // AllowNullKey *data.Bool `json:"allow_nullkey"` + // EncryptionEnabled *data.Bool `json:"encryption_enabled"` + // NotAllowAccessByTbl *data.Bool `json:"not_allow_access_by_tbl"` } func (i *BucketInfo) BucketName() string { @@ -48,6 +47,7 @@ func (i *BucketInfo) BucketName() string { return "" } + func (i *BucketInfo) DescriptionString() string { return fmt.Sprintf("%s", i.BucketName()) }