diff --git a/.golangci.yml b/.golangci.yml index 2e98e26..507e3bf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -70,5 +70,8 @@ issues: - linters: - goconst text: "string `audio` has" + - linters: + - goconst + text: "string `unknown` has" include: # - EXC0002 \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index bcc03cb..e103b75 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,6 +19,7 @@ "errcheck", "errorlint", "exportloopref", + "faststart", "funlen", "gifs", "gochecknoglobals", @@ -44,6 +45,7 @@ "luma", "lumaasset", "lumamattes", + "movflags", "nakedret", "noctx", "openapi", diff --git a/README.md b/README.md index 2143e40..5c7313e 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ If you would like to sponsor features, bugs or prioritization, reach out to one * 😎 Allow to use local file from `url` filed (`file:///Users/dblk/clips/my_asset`) * 😎 Add an endpoint `/dl/{version}/renders/:id` to download renders (instead of cdn/s3) * 😎 Add other value in resolution (`360`, `480`, `540`, `720`) all with default `25 fps`. +* 😎 Add destination to Youtube * [`Planned`] Allow to use ftp file from `url` filed (`ftp://user:password@dblk.org/mypath/my_asset`) -* [`Planned`] Add destination to Youtube ### Shotstack implementation progress @@ -82,7 +82,7 @@ At the end of the road this section should either disappear or be full of `Yes` | Output | range | Not yet | | | Output | poster | Not yet | | | Output | thumbnail | Not yet | | -| Output | destinations | Not yet | | +| Output | destinations | Partial 🛠 | `shotstack` won't be implemented. | #### Endpoint implementation diff --git a/api/openapi.yaml b/api/openapi.yaml index 9ce62c4..6de4985 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - description: "Shottowwer is the open source version of Shotstack which is a video, image and audio editing service that allows\ + description: "Shottower is the open source version of Shotstack which is a video, image and audio editing service that allows\ \ for the automated\ngeneration of videos, images and audio using JSON and a RESTful\ \ API.\n\nYou arrange and configure an edit and POST it to the API which will\ \ render your media and provide a file \nlocation when complete.\n\nFor more details\ @@ -1906,12 +1906,14 @@ components: anyOf: - $ref: '#/components/schemas/ShotstackDestination' - $ref: '#/components/schemas/MuxDestination' + - $ref: '#/components/schemas/YoutubeDestination' description: "A destination is a location where output files can be sent to\ \ for serving or hosting. By default all rendered assets are automatically\ \ sent to the [Shotstack hosting destination](https://shotstack.io/docs/guide/serving-assets/hosting).\ \ You can add other destinations to send assets to. The following destinations\ \ are available:\n " + \
  • DestinationMux
    • \n + \
  • DestinationYoutube
    • \n" discriminator: propertyName: destinations type: object @@ -1965,6 +1967,39 @@ components: - signed type: string type: array + YoutubeDestination: + description: "Send rendered videos to [Youtube](https://www.youtube.com/) video\ + \ hosting and streaming service. Add the `youtube` destination provider to send\ + \ the output video to Youtube. Youtube credentials are required and added inside Options for now in the\ + \ request." + properties: + provider: + default: youtube + description: The destination to send rendered assets to - set to `youtube` for + Youtube. + example: youtube + type: string + options: + $ref: '#/components/schemas/YoutubeDestinationOptions' + required: + - provider + type: object + YoutubeDestinationOptions: + description: Pass additional options to control how Youtube processes video. + properties: + title: + description: The title of the video + type: string + maxLength: 100 + description: + description: The description of the video + type: string + maxLength: 5000 + privacy: + description: The privacy of the video + enum: + - unlisted + type: string Template: description: "A template is a saved [Edit](#tocs_edit) than can be loaded and\ \ re-used." diff --git a/go/ffmpeg.go b/go/ffmpeg.go index 7cc4a9e..e581e83 100644 --- a/go/ffmpeg.go +++ b/go/ffmpeg.go @@ -37,18 +37,19 @@ type FFMPEGTrack struct { } type FFMPEG struct { - src []FFMPEGSource - defaultParams bool - tracks []FFMPEGTrack - size Size - format string - fps float32 - hasOverlay bool - backgroundColor string - overlayFillerCounter int - fillerCounter int - outputName string - duration float32 + src []FFMPEGSource + defaultParams bool + tracks []FFMPEGTrack + size Size + format string + fps float32 + hasOverlay bool + backgroundColor string + overlayFillerCounter int + fillerCounter int + outputName string + duration float32 + hasYoutubeDestination bool } func NewFFMPEGCommand() FFMPEGCommand { @@ -156,6 +157,11 @@ func (s *FFMPEG) ClipAudioMerge(sourceClip int, trackNumber int, clipNumber int, return clipEffect } +func (s *FFMPEG) HasYoutubeDestination() error { + s.hasYoutubeDestination = true + return nil +} + func (s *FFMPEG) SetOutputFormat(format string) error { s.format = format return nil @@ -696,6 +702,12 @@ func (s *FFMPEG) ToString() []string { parameters = append(parameters, "-vsync") // https://stackoverflow.com/questions/18064604/frame-rate-very-high-for-a-muxer-not-efficiently-supporting-it parameters = append(parameters, "2") + // Help youtube to re-encode before upload complete (https://trac.ffmpeg.org/wiki/Encode/H.264) + if s.hasYoutubeDestination { + parameters = append(parameters, "-movflags") + parameters = append(parameters, "+faststart") + } + var outputName = s.generateOutputName() parameters = append(parameters, outputName) diff --git a/go/model_destinations.go b/go/model_destinations.go index 761c388..de988e1 100644 --- a/go/model_destinations.go +++ b/go/model_destinations.go @@ -26,11 +26,59 @@ along with this program. If not, see . package openapi +import ( + "reflect" +) + +type DestinationProviderType int64 + +const ( + MuxDestinationType DestinationProviderType = iota + YoutubeDestinationType + UnknownDestinationType +) + +func (s DestinationProviderType) String() string { + switch s { // nolint:exhaustive + case MuxDestinationType: + return "mux" + case YoutubeDestinationType: + return "youtube" + + default: + return "unknown" + } +} + func NewDestination(provider string, obj map[string]interface{}) interface{} { - switch provider { + switch provider { // nolint:exhaustive case "mux": return NewMuxDestination(obj) + case "youtube": + return NewYoutubeDestination(obj) } return nil } + +func GetDestinationProvider(destination interface{}) DestinationProviderType { + switch reflect.TypeOf(destination).String() { + case "*openapi.MuxDestination": + return MuxDestinationType + case "*openapi.YoutubeDestination": + return YoutubeDestinationType + default: + return UnknownDestinationType + } +} + +// AssertDestinationsRequired checks if the required fields are not zero-ed +func AssertDestinationsRequired(obj interface{}) error { + switch GetDestinationProvider(obj) { // nolint:exhaustive + case YoutubeDestinationType: + return AssertYoutubeDestinationRequired(obj.(*YoutubeDestination)) + case MuxDestinationType: + return AssertMuxDestinationRequired(obj.(*MuxDestination)) + } + return nil +} diff --git a/go/model_ffmpeg.go b/go/model_ffmpeg.go index 49951a0..55c7a41 100644 --- a/go/model_ffmpeg.go +++ b/go/model_ffmpeg.go @@ -50,5 +50,6 @@ type FFMPEGCommand interface { ToFFMPEG(*RenderQueue, *ProcessingQueue) error GetOutputName() string GetDuration() float32 + HasYoutubeDestination() error OverlayAllTracks([]string) string } diff --git a/go/model_mux_destination.go b/go/model_mux_destination.go index e8c548b..acf9274 100644 --- a/go/model_mux_destination.go +++ b/go/model_mux_destination.go @@ -35,20 +35,20 @@ type MuxDestination struct { Options MuxDestinationOptions `json:"options,omitempty"` } -func NewMuxDestination(obj map[string]interface{}) interface{} { +func NewMuxDestination(obj map[string]interface{}) *MuxDestination { destination := &MuxDestination{ Provider: obj["provider"].(string), } if obj["options"] != nil { - destination.Options = obj["options"].(MuxDestinationOptions) + destination.Options = *NewMuxDestinationOptions(obj["options"].(map[string]interface{})) } return destination } // AssertMuxDestinationRequired checks if the required fields are not zero-ed -func AssertMuxDestinationRequired(obj MuxDestination) error { +func AssertMuxDestinationRequired(obj *MuxDestination) error { elements := map[string]interface{}{ "provider": obj.Provider, } @@ -72,6 +72,6 @@ func AssertRecurseMuxDestinationRequired(objSlice interface{}) error { if !ok { return ErrTypeAssertionError } - return AssertMuxDestinationRequired(aMuxDestination) + return AssertMuxDestinationRequired(&aMuxDestination) }) } diff --git a/go/model_mux_destination_options.go b/go/model_mux_destination_options.go index 1006d3d..30f176d 100644 --- a/go/model_mux_destination_options.go +++ b/go/model_mux_destination_options.go @@ -33,6 +33,19 @@ type MuxDestinationOptions struct { PlaybackPolicy []string `json:"playbackPolicy,omitempty"` } +func NewMuxDestinationOptions(obj map[string]interface{}) *MuxDestinationOptions { + options := &MuxDestinationOptions{} + + if obj["playbackPolicy"] != nil { + policy := obj["playbackPolicy"].([]interface{}) + for _, i := range policy { + options.PlaybackPolicy = append(options.PlaybackPolicy, i.(string)) + } + } + + return options +} + // AssertMuxDestinationOptionsRequired checks if the required fields are not zero-ed func AssertMuxDestinationOptionsRequired(obj MuxDestinationOptions) error { return nil diff --git a/go/model_output.go b/go/model_output.go index e618375..733e735 100644 --- a/go/model_output.go +++ b/go/model_output.go @@ -69,7 +69,7 @@ type Output struct { Destinations []interface{} `json:"destinations,omitempty"` } -func NewOutput(data map[string]interface{}) *Output { +func NewOutput(data map[string]interface{}, destinations []interface{}) *Output { output := &Output{ Format: data["format"].(string), } @@ -106,13 +106,7 @@ func NewOutput(data map[string]interface{}) *Output { *output.Thumbnail = *NewThumbnail(data["thumbnail"].(map[string]interface{})) } - if data["destinations"] != nil { - for _, dest := range data["destinations"].([]map[string]interface{}) { - var provider = dest["provider"].(string) - destination := NewDestination(provider, dest) - output.Destinations = append(output.Destinations, destination) - } - } + output.Destinations = destinations return output } @@ -123,7 +117,16 @@ func (s *Output) UnmarshalJSON(data []byte) error { return err } - *s = *NewOutput(obj) + var destinations []interface{} + if obj["destinations"] != nil { + for _, dest := range obj["destinations"].([]interface{}) { + var provider = dest.(map[string]interface{})["provider"].(string) + destination := NewDestination(provider, dest.(map[string]interface{})) + destinations = append(destinations, destination) + } + } + + *s = *NewOutput(obj, destinations) return nil } @@ -193,11 +196,13 @@ func AssertOutputRequired(obj *Output) error { if err := AssertThumbnailRequired(obj.Thumbnail); err != nil { return err } - // for i := range obj.Destinations { - // if err := AssertDestinationsRequired(&obj.Destinations[i]); err != nil { - // return err - // } - // } + if obj.Destinations != nil { + for i := range obj.Destinations { + if err := AssertDestinationsRequired(obj.Destinations[i]); err != nil { + return err + } + } + } return nil } diff --git a/go/model_youtube_destination.go b/go/model_youtube_destination.go new file mode 100644 index 0000000..cca3153 --- /dev/null +++ b/go/model_youtube_destination.go @@ -0,0 +1,77 @@ +/* +shottower +Copyright (C) 2022 Rémy Boulanouar + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +/* + * Shottower + * + * Shottower is the open source version of Shotstack which is a video, image and audio editing service that allows for the automated generation of videos, images and audio using JSON and a RESTful API. You arrange and configure an edit and POST it to the API which will render your media and provide a file location when complete. For more details visit [shottower](https://github.com/DblK/shottower) or checkout our [getting started](https://shotstack.io/docs/guide/) documentation. There are two main API's, one for editing and generating assets (Edit API) and one for managing hosted assets (Serve API). The Edit API base URL is: http://0.0.0.0:4000/{version} The Serve API base URL is: http://0.0.0.0:4000/serve/{version} + * + * API version: stage + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +// YoutubeDestination - Send rendered videos to [Youtube](https://www.youtube.com/) video hosting and streaming service. Add the `youtube` destination provider to send the output video to Youtube. Youtube credentials are required and added inside Options for now in the request. +type YoutubeDestination struct { + + // The destination to send rendered assets to - set to `youtube` for Youtube. + Provider string `json:"provider"` + + Options YoutubeDestinationOptions `json:"options,omitempty"` +} + +func NewYoutubeDestination(obj map[string]interface{}) *YoutubeDestination { + destination := &YoutubeDestination{ + Provider: obj["provider"].(string), + } + + if obj["options"] != nil { + destination.Options = *NewYoutubeDestinationOptions(obj["options"].(map[string]interface{})) + } + + return destination +} + +// AssertYoutubeDestinationRequired checks if the required fields are not zero-ed +func AssertYoutubeDestinationRequired(obj *YoutubeDestination) error { + elements := map[string]interface{}{ + "provider": obj.Provider, + } + for name, el := range elements { + if isZero := IsZeroValue(el); isZero { + return &RequiredError{Field: name} + } + } + + if err := AssertYoutubeDestinationOptionsRequired(obj.Options); err != nil { + return err + } + return nil +} + +// AssertRecurseYoutubeDestinationRequired recursively checks if required fields are not zero-ed in a nested slice. +// Accepts only nested slice of YoutubeDestination (e.g. [][]YoutubeDestination), otherwise ErrTypeAssertionError is thrown. +func AssertRecurseYoutubeDestinationRequired(objSlice interface{}) error { + return AssertRecurseInterfaceRequired(objSlice, func(obj interface{}) error { + aYoutubeDestination, ok := obj.(YoutubeDestination) + if !ok { + return ErrTypeAssertionError + } + return AssertYoutubeDestinationRequired(&aYoutubeDestination) + }) +} diff --git a/go/model_youtube_destination_options.go b/go/model_youtube_destination_options.go new file mode 100644 index 0000000..ec6a0b8 --- /dev/null +++ b/go/model_youtube_destination_options.go @@ -0,0 +1,73 @@ +/* +shottower +Copyright (C) 2022 Rémy Boulanouar + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +/* + * Shottower + * + * Shottower is the open source version of Shotstack which is a video, image and audio editing service that allows for the automated generation of videos, images and audio using JSON and a RESTful API. You arrange and configure an edit and POST it to the API which will render your media and provide a file location when complete. For more details visit [shottower](https://github.com/DblK/shottower) or checkout our [getting started](https://shotstack.io/docs/guide/) documentation. There are two main API's, one for editing and generating assets (Edit API) and one for managing hosted assets (Serve API). The Edit API base URL is: http://0.0.0.0:4000/{version} The Serve API base URL is: http://0.0.0.0:4000/serve/{version} + * + * API version: stage + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +// YoutubeDestinationOptions - Pass additional options to control how Youtube processes video. +type YoutubeDestinationOptions struct { + + // The title of the video + Title string `json:"title,omitempty"` + + // The description of the video + Description string `json:"description,omitempty"` + + // The privacy of the video + Privacy string `json:"privacy,omitempty"` +} + +func NewYoutubeDestinationOptions(obj map[string]interface{}) *YoutubeDestinationOptions { + options := &YoutubeDestinationOptions{} + + if obj["title"] != nil { + options.Title = obj["title"].(string) + } + if obj["description"] != nil { + options.Description = obj["description"].(string) + } + if obj["privacy"] != nil { + options.Privacy = obj["privacy"].(string) + } + + return options +} + +// AssertYoutubeDestinationOptionsRequired checks if the required fields are not zero-ed +func AssertYoutubeDestinationOptionsRequired(obj YoutubeDestinationOptions) error { + return nil +} + +// AssertRecurseYoutubeDestinationOptionsRequired recursively checks if required fields are not zero-ed in a nested slice. +// Accepts only nested slice of YoutubeDestinationOptions (e.g. [][]YoutubeDestinationOptions), otherwise ErrTypeAssertionError is thrown. +func AssertRecurseYoutubeDestinationOptionsRequired(objSlice interface{}) error { + return AssertRecurseInterfaceRequired(objSlice, func(obj interface{}) error { + aYoutubeDestinationOptions, ok := obj.(YoutubeDestinationOptions) + if !ok { + return ErrTypeAssertionError + } + return AssertYoutubeDestinationOptionsRequired(aYoutubeDestinationOptions) + }) +} diff --git a/go/queue.go b/go/queue.go index e705b9f..a5ace09 100644 --- a/go/queue.go +++ b/go/queue.go @@ -84,10 +84,8 @@ func (s *ProcessingQueue) FindSourceClip(trackNumber int, clipNumber int) string return "" } -func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) { +func (s *ProcessingQueue) checkAndTakeJob(editAPI EditAPIServicer) { var queue = editAPI.GetQueue() - fmt.Println("ProcessQueue (pending:", len(editAPI.GetQueuePending()), ") ", len(queue)) - if len(queue) != 0 && s.currentQueue == nil { if len(editAPI.GetQueuePending()) != 0 { s.currentQueue = editAPI.GetQueuePending()[0] @@ -105,34 +103,55 @@ func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) { go s.FetchAssets(s.currentQueue) } } +} - if s.currentQueue != nil { - fmt.Println(s.currentQueue.Status.String()) - - if s.currentQueue.InternalStatus == Fetched { - params := s.GenerateParameters(s.currentQueue) - s.currentQueue.Updated = time.Now() +func (s *ProcessingQueue) processingRender() { + fmt.Println(s.currentQueue.Status.String()) - // Launch Rendering - // fmt.Println(params) - go s.ExecuteFFMpeg(params) + if s.currentQueue.InternalStatus == Fetched { + // Optimize output for youtube + for _, destination := range s.currentQueue.Data.Output.Destinations { + if GetDestinationProvider(destination) == YoutubeDestinationType { + _ = s.currentQueue.FFMPEGCommand.HasYoutubeDestination() + } } - if s.currentQueue.InternalStatus == Rendered { - s.currentQueue.Status = Saving - s.currentQueue.InternalStatus = Saving - s.currentQueue.FileName = s.currentQueue.FFMPEGCommand.GetOutputName() - } + params := s.GenerateParameters(s.currentQueue) + s.currentQueue.Updated = time.Now() - if s.currentQueue.InternalStatus == Saving { - // FIXME: Pass through Saving status at some point + // Launch Rendering + // fmt.Println(params) + go s.ExecuteFFMpeg(params) + } + + if s.currentQueue.InternalStatus == Rendered { + s.currentQueue.Status = Saving + s.currentQueue.InternalStatus = Saving + s.currentQueue.FileName = s.currentQueue.FFMPEGCommand.GetOutputName() + } + + if s.currentQueue.InternalStatus == Saving { + if s.currentQueue.Data.Output.Destinations != nil { + go s.SendToDestinations(s.currentQueue.Data.Output.Destinations, s.currentQueue.FileName) + } else { s.currentQueue.Status = Done s.currentQueue.InternalStatus = Done } + } - if s.currentQueue.InternalStatus == Failed || s.currentQueue.InternalStatus == Done { - s.currentQueue = nil - } + if s.currentQueue.InternalStatus == Failed || s.currentQueue.InternalStatus == Done { + s.currentQueue = nil + } +} + +func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) { + var queue = editAPI.GetQueue() + fmt.Println("ProcessQueue (pending:", len(editAPI.GetQueuePending()), ") ", len(queue)) + + s.checkAndTakeJob(editAPI) + + if s.currentQueue != nil { + s.processingRender() } go time.AfterFunc(1*time.Second, func() { @@ -140,6 +159,26 @@ func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) { }) } +func (s *ProcessingQueue) SendToDestinations(destinations []interface{}, fileName string) { + for _, destination := range destinations { + provider := GetDestinationProvider(destination) + switch provider { // nolint:exhaustive + case YoutubeDestinationType: + fmt.Println("sending to youtube", fileName) + fmt.Println(destination) + case MuxDestinationType: + // TODO: Do it later + // fmt.Println("sending to mux", fileName) + // fmt.Println(destination) + default: + fmt.Println("Destination not handled", provider) + } + } + + s.currentQueue.Status = Done + s.currentQueue.InternalStatus = Done +} + func (s *ProcessingQueue) ExecuteFFMpeg(params []string) { s.currentQueue.InternalStatus = Rendering cmd := exec.Command("ffmpeg", params...)