diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index eca1caa..533681a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -29,6 +29,13 @@ RUN mkdir -p /home/vscode/.local/bin && \ chmod +x /home/vscode/.local/bin/yt-dlp && \ chown -R vscode /home/vscode/.local +# Enables preservation of bash history +RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" && \ + mkdir /commandhistory && \ + touch /commandhistory/.bash_history && \ + chown -R $USERNAME /commandhistory && \ + echo "$SNIPPET" >> "/home/$USERNAME/.bashrc" + USER $USERNAME ENTRYPOINT ["/bin/bash"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cdcbc83..73139e5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,6 @@ "python.pythonPath": "/usr/local/bin/python", "python.languageServer": "Pylance", "python.globalModuleInstallation": false, - "python.logging.level": "debug", "python.venvPath": "~/.pyenv", "python.terminal.activateEnvInCurrentTerminal": true, "python.analysis.autoSearchPaths": true, @@ -41,27 +40,40 @@ "terminal.integrated.gpuAcceleration": "auto", "terminal.integrated.useWslProfiles": true, "terminal.integrated.defaultProfile.linux":"bash" - } + }, + "extensions": [ + "ms-python.python" + ] }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", ] }, "features": { "ghcr.io/devcontainers/features/docker-outside-of-docker:1.2.1": {}, - "ghcr.io/devcontainers/features/git:1.1.5": {} + "ghcr.io/devcontainers/features/git:1.1.5": {}, + "ghcr.io/devcontainers-contrib/features/twine:2": {}, + "ghcr.io/devcontainers-contrib/features/mypy:2": {}, + "ghcr.io/devcontainers-contrib/features/isort:2": {}, + "ghcr.io/devcontainers-contrib/features/flake8:2": {}, + "ghcr.io/devcontainers-contrib/features/black:2": {}, + "ghcr.io/devcontainers-contrib/features/bandit:2": {} }, + "mounts": [ + "source=projectname-bashhistory,target=/commandhistory,type=volume" + ], + // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "/scripts/postCreateCommand.sh", + "postCreateCommand": "/bin/bash dependencies/scripts/postCreateCommand.sh", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" diff --git a/.dockerignore b/.dockerignore index 0180a78..d7ea92f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ final/* -config/auth.py .env client_secret.json cookies.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 22d6177..c9d9a21 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,16 +13,16 @@ jobs: - name: Run ttsvibelounge Script env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RYBO_POLLY_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + RYBO_POLLY_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} CREDENTIALS_STORAGE: ${{ secrets.CREDENTIALS_STORAGE }} - PRAW_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} - PRAW_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} - PRAW_USER_AGENT: ${{ secrets.PRAW_USER_AGENT }} - PRAW_USERNAME: ${{ secrets.PRAW_USERNAME }} - PRAW_PASSWORD: ${{ secrets.PRAW_PASSWORD }} - RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} - RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} + RYBO_REDDIT_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} + RYBO_REDDIT_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} + RYBO_REDDIT_USER_AGENT: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0 + RYBO_REDDIT_USERNAME: ${{ secrets.PRAW_USERNAME }} + RYBO_REDDIT_PASSWORD: ${{ secrets.PRAW_PASSWORD }} + RYBO_RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} + RYBO_RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} YOUTUBE_CLIENT_SECRET: ${{ secrets.YOUTUBE_CLIENT_SECRET }} GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | @@ -31,12 +31,13 @@ jobs: echo $GITHUB_WORKSPACE echo $YOUTUBE_CLIENT_SECRET > client_secret.json echo $CREDENTIALS_STORAGE > credentials.storage - cp config/auth-env.py config/auth.py + pip install -r requirements.txt playwright install - python3 app.py --total-posts 1 \ - --enable-background \ - --background-directory /app/assets/backgrounds \ - --enable-mentions + pip install --editable . + rybo --total-posts 1 \ + --enable-background \ + --background-directory /app/assets/backgrounds \ + --enable-mentions python3 refresh_token.py rm -f client_secret.json rm -f credentials.storage diff --git a/.github/workflows/tssvibelounge.yml b/.github/workflows/tssvibelounge.yml index e4141be..1d37de4 100644 --- a/.github/workflows/tssvibelounge.yml +++ b/.github/workflows/tssvibelounge.yml @@ -29,16 +29,16 @@ jobs: - name: Run ttsvibelounge Script env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RYBO_POLLY_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + RYBO_POLLY_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} CREDENTIALS_STORAGE: ${{ secrets.CREDENTIALS_STORAGE }} - PRAW_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} - PRAW_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} - PRAW_USER_AGENT: ${{ secrets.PRAW_USER_AGENT }} - PRAW_USERNAME: ${{ secrets.PRAW_USERNAME }} - PRAW_PASSWORD: ${{ secrets.PRAW_PASSWORD }} - RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} - RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} + RYBO_REDDIT_CLIENT_ID: ${{ secrets.PRAW_CLIENT_ID }} + RYBO_REDDIT_CLIENT_SECRET: ${{ secrets.PRAW_CLIENT_SECRET }} + RYBO_REDDIT_USER_AGENT: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0 + RYBO_REDDIT_USERNAME: ${{ secrets.PRAW_USERNAME }} + RYBO_REDDIT_PASSWORD: ${{ secrets.PRAW_PASSWORD }} + RYBO_RUMBLE_PASSWORD: ${{ secrets.RUMBLE_PASSWORD }} + RYBO_RUMBLE_USERNAME: ${{ secrets.RUMBLE_USERNAME }} YOUTUBE_CLIENT_SECRET: ${{ secrets.YOUTUBE_CLIENT_SECRET }} GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | @@ -47,18 +47,18 @@ jobs: echo $GITHUB_WORKSPACE echo $YOUTUBE_CLIENT_SECRET > client_secret.json echo $CREDENTIALS_STORAGE > credentials.storage - cp config/auth-env.py config/auth.py + pip install -r requirements.txt playwright install - python3 app.py --total-posts 1 \ - --enable-upload \ - --enable-background \ - --background-directory /app/assets/backgrounds \ - --enable-mentions + pip install --editable . + rybo --total-posts 1 \ + --enable-upload \ + --enable-background \ + --background-directory /app/assets/backgrounds \ + --enable-mentions python3 refresh_token.py rm -f client_secret.json rm -f credentials.storage - - name: check for changes run: | git config --global --add safe.directory $(realpath .) diff --git a/.gitignore b/.gitignore index edf10ce..1ad4096 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,9 @@ client_secret.json cookies*.json credentials.storage videos/ -config/auth.py .env +venv +*.egg-info **/__pycache__/ assets/work_dir/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c31efd8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Alex Laverty + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index fc1bb1e..10a4d9c 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,59 @@ # Automated Reddit to Youtube Bot -* [Description](#description) -* [Example Videos](#example-videos) -* [Install Prerequisite Components](#install-prerequisite-components) -* [Git clone repository](#git-clone-repository) -* [Generate Reddit Tokens](#generate-reddit-tokens) -* [Copy auth config](#copy-auth-config) -* [Python Pip Install Dependencies](#python-pip-install-dependencies) -* [Install Playwright](#install-playwright) -* [Run Python Script](#run-python-script) -* [Generate a Video for a Specific Post](#generate-a-video-for-a-specific-post) -* [Generate Only Thumbnails](#generate-only-thumbnails) -* [Enable a Newscaster](#enable-a-newscaster) -* [Settings.py File](#settings.py-file) - +* [Description](#Description) +* [Example Videos](#ExampleVideos) +* [Windows](#Windows) + * [ Install Prerequisite Components](#InstallPrerequisiteComponents) + * [ Clone the Git Repository](#ClonetheGitRepository) + * [ Configure a Virtual Environment](#ConfigureaVirtualEnvironment) + * [ Configure Playwright](#ConfigurePlaywright) + * [ Install the pip package](#Installthepippackage) + * [ Generate a Reddit token](#GenerateaReddittoken) + * [ Set up your credentials](#Setupyourcredentials) + * [ Run the CLI utility](#RuntheCLIutility) +* [Configuration](#Configuration) + * [Configuring the CLI](#ConfiguringtheCLI) + * [ Downloading video backgrounds using yt-dlp](#Downloadingvideobackgroundsusingyt-dlp) + * [ Help](#Help) +* [Customising](#Customising) + * [ Specify Subreddits to Scrape](#SpecifySubredditstoScrape) + * [ Exclude Subreddits](#ExcludeSubreddits) + * [ Filter Reddit submissions by keyword](#FilterRedditsubmissionsbykeyword) + * [ Change the Text to Speech Engine](#ChangetheTexttoSpeechEngine) + * [ Limit the number of generated videos](#Limitthenumberofgeneratedvideos) + * [ Skip Reddit posts by submission score](#SkipRedditpostsbysubmissionscore) + * [ Filter Reddit posts by title length](#FilterRedditpostsbytitlelength) + * [ Filter Reddit posts by self text length](#FilterRedditpostsbyselftextlength) + * [ Filter Reddit posts by comment count](#FilterRedditpostsbycommentcount) + * [Limit number of posts to process](#Limitnumberofpoststoprocess) + * [ Configure number of thumbnails to generate](#Configurenumberofthumbnailstogenerate) + * [ Set the maximum video length](#Setthemaximumvideolength) + * [ Set maximum number of comments to include](#Setmaximumnumberofcommentstoinclude) + * [ Specify folder paths](#Specifyfolderpaths) + * [ Set video dimensions](#Setvideodimensions) + * [ Skip video compilation](#Skipvideocompilation) + * [ Skip YouTube uploading](#SkipYouTubeuploading) + * [ Add a video overlay](#Addavideooverlay) + * [ Add a Newscaster](#AddaNewscaster) + * [ Add a pause after each TTS file](#AddapauseaftereachTTSfile) + * [Modify appearnace of text](#Modifyappearnaceoftext) + * [ Download images from Lexica](#DownloadimagesfromLexica) +* [ Tips and Tricks](#TipsandTricks) + * [Generate a Video for a Specific Post](#GenerateaVideoforaSpecificPost) + * [Generate Only Thumbnails](#GenerateOnlyThumbnails) + * [Enable a Newscaster](#EnableaNewscaster) -## Description +## Description Scrape posts from Reddit and automatically generate Youtube Videos and Thumbnails -## Example Videos +## Example Videos Checkout my Youtube Channel for example videos made by this repo : @@ -37,70 +65,150 @@ Checkout my Youtube Channel for example videos made by this repo : # Quickstart Guide -# Windows - +## Windows [Watch the Python Reddit Youtube Bot Tutorial Video :](https://youtu.be/LaFFU9EskfA) [![Watch the video](assets/images/python-reddit-youtube-bot-tutorial.png)](https://youtu.be/LaFFU9EskfA) -## Install Prerequisite Components +### Install Prerequisite Components Install these prerequisite components first : * Git - https://git-scm.com/download/win -* Python 3.10 - https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe +* Python 3.11 - https://www.python.org/ftp/python/3.11.3/python-3.11.3-amd64.exe * Microsoft C++ Build Tools - https://visualstudio.microsoft.com/visual-cpp-build-tools/ * ImageMagick - https://imagemagick.org/script/download.php#windows -## Git clone repository +### Clone the Git Repository -``` -git clone git@github.com:alexlaverty/python-reddit-youtube-bot.git -cd python-reddit-youtube-bot +```powershell +> git clone git@github.com:alexlaverty/python-reddit-youtube-bot.git +> cd python-reddit-youtube-bot ``` -## Generate Reddit Tokens +### Configure a Virtual Environment -Generate Reddit PRAW Tokens - https://www.reddit.com/prefs/apps/ +Create a virtual environment, and install package dependencies : -## Copy auth config +```powershell +> python -m venv venv +... +> .\venv\Scripts\activate.ps1 +... +> pip install -r requirements.txt +Collecting boto3==1.26.123 + Using cached boto3-1.26.123-py3-none-any.whl (135 kB) +Collecting bs4==0.0.1 + Using cached bs4-0.0.1.tar.gz (1.1 kB) + Preparing metadata (setup.py) ... done +... +``` -Create a copy of the auth-example.py file and name it auth.py : +### Configure Playwright -``` -copy config/auth-example.py config/auth.py +Install and configure playwright by running : + +```powershell +> playwright install ``` -Update the `auth.py` file to contain the Reddit Auth tokens you generated in the previous step. +### Install the pip package -## Python Pip Install Dependencies +Install the command-line utility used to run the bot : -``` -pip install -r requirements.txt +```powershell +> pip install --user --editable . +Obtaining file:///workspaces/python-reddit-youtube-bot-forked + ... +Building wheels for collected packages: rybo + ... +Successfully built rybo +Installing collected packages: rybo + ... +Successfully installed rybo-0.0.1 ``` -## Install Playwright +### Generate a Reddit token -Install and configure playwright by running : +Generate Reddit OAuth credentials via https://www.reddit.com/prefs/apps/. -``` -playwright install -``` +### Set up your credentials -## Run Python Script +Add the Reddit token via environment variables, or in a YAML configuration +file. The default expected location for this file is `$HOME/rybo.yaml`. -Run the python script : +| Environment Variable | Configuration File Option | Description | +|--------------------------------|---------------------------|------------------------------------------------------------| +| `RYBO_REDDIT_CLIENT_ID` | `reddit.client_id` | Client id used to authenticate against the Reddit API. | +| `RYBO_REDDIT_CLIENT_SECRET` | `reddit.client_secret` | Client secret used to authenticate against the Reddit API. | +| `RYBO_REDDIT_USERNAME` | `reddit.username` | Username used to log in to the Reddit Web UI. | +| `RYBO_REDDIT_PASSWORD` | `reddit.password` | Password used to log in to the Reddit Web UI. | +| `RYBO_POLLY_ACCESS_KEY` | `polly.access_key` | AWS Access Key used to interact with the Polly service. | +| `RYBO_POLLY_SECRET_ACCESS_KEY` | `polly.secret_access_key` | AWS secret used to interact with the Polly service. | +| `RYBO_RUMBLE_USERNAME` | `rumble.username` | Username used to interact with the Rumble Web UI. | +| `RYBO_RUMBLE_PASSWORD` | `rumble.password` | Password used to interact with the Rumble Web UI. | +### Run the CLI utility + +```powershell +> rybo ``` -python app.py -``` -when it completes the video will be generated into the `videos` folder and will be named `final.mp4` +When it completes, the video will be generated into the `videos` folder and +will be named `final.mp4`. + +## Configuration + +### Configuring the CLI + +Configuration options can be specified via CLI parameters, environment variables, or a +yaml configuration file. + +The order of precedence is as follows: -# Downloading video backgrounds using yt-dlp : +1. Yaml configuration file (default location: `$HOME/rybo.yaml`) +2. Environment variables. +3. User-specified CLI parameters. + +For example, if `disable_overlay: True` is set in `$HOME/rybo.yaml`, and +the `RYBO_DISABLE_OVERLAY=False` environment variable is also set, then +video overlay will be disabled (false) in the runtime configuration. + +
+Click to view the list of configuration options + +| Argument | Environment Variable | Configuration File Option | Default Value | Description | +|--------------------------|--------------------------------|---------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-h`/`--help` | | | | Display the help menu. | +| `--config` | `RYBO_CONFIG` | | `$HOME/rybo.yaml` | Load settings from a configuration file. | +| `--version ` | | | | Display version information. | +| `--background-directory` | `RYBO_BACKGROUND_DIRECTORY` | `background_directory` | `assets/backgrounds` | Folder path to videos that will be used for the video background. | +| `--comment-style` | `RYBO_COMMENT_STYLE` | `comment_style` | `reddit` | Use text based, or image based Reddit comments. Choices are **text** or **image**. | +| `--disable-overlay` | `RYBO_DISABLE_OVERLAY` | `disable_overlay` | `False` | Enable or disable the video overlay. | +| `--disable-selftext` | `RYBO_DISABLE_SELFTEXT` | `disable_selftext` | `False` | Enable or disable self-text video generation. | +| `--enable-background` | `RYBO_ENABLE_BACKGROUND` | `enable_background` | `False` | Enable or disable adding a background to the video. | +| `--enable-mentions` | `RYBO_ENABLE_MENTIONS` | `enable_mentions` | `False` | Check the reddit account for user mentions. | +| `--enable-nsfw` | `RYBO_ENABLE_NSFW` | `enable_nsfw` | `False` | Include, or ignore posts tagged as Not Safe for Work. | +| `--enable-upload` | `RYBO_ENABLE_UPLOAD` | `enable_upload` | `False` | Enable or disable uploading videos to YouTube. | +| `--orientation` | `RYBO_ORIENTATION` | `orientation` | `landscape` | Set the video orientation. Choices are **portrait** or **Landscape**. | +| `--shorts` | `RYBO_SHORTS` | `shorts` | `False` | Enable or disable generating a YouTube shorts video. | +| `--sort` | `RYBO_SORT` | `sort` | `hot` | Set the sorting order when scanning Reddit posts. Choices are **top** or **hot**. | +| `--submission-score` | `RYBO_SUBMISSION_SCORE` | `submission_score` | `5000` | Minimum submission score threshold. | +| `--subreddits` | `RYBO_SUBREDDITS` | `subreddits` | | List of subreddits to scan, where each subreddit is separated with a **+**. | +| `--story-mode` | `RYBO_STORY_MODE` | `story_mode` | `False` | Enable or disable video generation for the post title and selftext only, disables user comments. | +| `--thumbnail-only` | `RYBO_THUMBNAIL_ONLY` | `thumbnail_only` | `False` | Enable or disable generation of just the video thumbnails. | +| `--time` | `RYBO_TIME` | `time` | `day` | Filter Reddit submissions by time. Choices are **all**, **day**, **hour**, **month**, **week** or **year**. | +| `--total-posts` | `RYBO_TOTAL_POSTS` | `total_posts` | `10` | Total number of reddit submissions to process. | +| `--url` | `RYBO_URL` | `url` | | Generate a video for a single Reddit submission. | +| `--video-length` | `RYBO_VIDEO_LENGTH` | `video_length` | `600` | Sets how long the generated video will be, in seconds. | +| `--voice-engine` | `RYBO_VOICE_ENGINE` | `voice_engine` | `edge-tts` | Specify which text-to-speech engine should be used to narrate the video. Choices are **polly**, **balcon**, **gtts**, **tiktok**, **edge-tts**, **streamlabspolly**. | + +
+ +### Downloading video backgrounds using yt-dlp If you want to add a video background then install yt-dlp : @@ -114,75 +222,74 @@ cd assets/backgrounds yt-dlp --playlist-items 1:10 -f 22 --output "%(uploader)s_%(id)s.%(ext)s" https://www.youtube.com/playlist?list=PLGmxyVGSCDKvmLInHxJ9VdiwEb82Lxd2E ``` -# Help +### Help You can view available parameters by passing in `--help` : -``` -python app.py --help +```powershell +rybo --help + +============================== YOUTUBE REDDIT BOT =============================== +OS Version : Linux 5.15.90.1-microsoft-standard-WSL2 +Python Version : 3.11.3 (main, May 4 2023, 05:53:32) [GCC 10.2.1 20210110] +Rybo Version : 0.0.1 + +usage: rybo [-h] [--config CONFIG] [--version] [--background-directory] [-c {text,reddit}] [-o] [--disable-selftext] [-b] [--enable-mentions] [-n] [-p] + [--orientation {landscape,portrait}] [--shorts] [--sort {top,hot}] [--submission-score SUBMISSION_SCORE] [--subreddits SUBREDDITS] [-s] [-t] + [--time {all,day,hour,month,week,year}] [--total-posts TOTAL_POSTS] [-u URL] [-l VIDEO_LENGTH] + [--voice-engine {polly,balcon,gtts,tiktok,edge-tts,streamlabspolly}] -##### YOUTUBE REDDIT BOT ##### -usage: app.py [-h] [-l VIDEO_LENGTH] [-o] [-s] [-t] [-u URL] +Generate vidoes from reddit posts. options: -h, --help show this help message and exit - -l VIDEO_LENGTH, --video-length VIDEO_LENGTH - Set how long you want the video to be + --config CONFIG Path to the configuration file. + --version show programs version number and exit + --background-directory + Folder path to video backgrounds. + -c {text,reddit}, --comment-style {text,reddit} + Specify text based or reddit image comments. -o, --disable-overlay - Disable video overlay - -s, --story-mode Generate video for post title and selftext only, disables user comments - -t, --thumbnail-only Generate thumbnail image only + Disable video overlay. + --disable-selftext Disable selftext video generation. + -b, --enable-background + Enable video backgrounds. + --enable-mentions Check reddit account for u mentions. + -n, --enable-nsfw Allow NSFW Content. + -p, --enable-upload Upload video to youtube,requires client_secret.json and credentials. storage to be valid. + --orientation {landscape,portrait} + Sort Reddit posts by. + --shorts Generate Youtube Shorts Video. + --sort {top,hot} Sort Reddit posts by. + --submission-score SUBMISSION_SCORE + Minimum submission score threshold. + --subreddits SUBREDDITS + Specify Subreddits, seperate with +. + -s, --story-mode Generate video for post title and selftext only, disables user comments. + -t, --thumbnail-only Generate thumbnail image only. + --time {all,day,hour,month,week,year} + Filter by time. + --total-posts TOTAL_POSTS + Enable video backgrounds. -u URL, --url URL Specify Reddit post url, seperate with a comma for multiple posts. + -l VIDEO_LENGTH, --video-length VIDEO_LENGTH + Set how long you want the video to be. + --voice-engine {polly,balcon,gtts,tiktok,edge-tts,streamlabspolly} + Specify which text to speech engine to use. ``` -## Generate a Video for a Specific Post - -or if you want to generate a video for a specific reddit post you can specify it via the `--url` param : - -``` -python app.py --url https://www.reddit.com/r/AskReddit/comments/hvsxty/which_legendary_reddit_post_comment_can_you_still/ -``` - -or you can do multiple url's by seperating with a comma, ie : - -``` -python app.py --url https://www.reddit.com/r/post1,https://www.reddit.com/r/post2,https://www.reddit.com/r/post3 -``` - -## Generate Only Thumbnails - -if you want to generate only thumbnails you can specify `--thumbnail-only` mode, this will skip video compilation process : - -``` -python app.py --thumbnail-only -``` - -## Enable a Newscaster - -If you want to enable a Newscaster, edit settings.py and set : - -``` -enable_newscaster = True -``` - -![](assets/newscaster.png) - -If the newcaster video has a green screen you can remove it with the following settings, -use an eye dropper to get the RGB colour of the greenscreen and set it to have it removed : +## Customising -``` -newscaster_remove_greenscreen = True -newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour -newscaster_greenscreen_remove_threshold = 100 -``` +Theres quite a few options you can customise in the `settings.py` file. -## Settings.py File +These will at some point be moved into the main `rybo.yaml` configuration file. -Theres quite a few options you can customise in the `settings.py` file : +
+Click to view all available customisation options -Specify which subreddits you want to scrape : +### Specify Subreddits to Scrape -``` +```python subreddits = [ "AmItheAsshole", "antiwork", @@ -200,21 +307,23 @@ subreddits = [ ] ``` -Subreddits to exclude : +### Exclude Subreddits -``` +```python subreddits_excluded = [ "r/CFB", ] ``` -Filter out reddit posts via specified keywords +### Filter Reddit submissions by keyword -``` +```python banned_keywords =["my", "nasty", "keywords"] ``` -Change the Text to Speech engine you want to use, note AWS Polly requires and AWS account and auth tokens and can incur costs : +### Change the Text to Speech Engine + +note AWS Polly requires and AWS account and auth tokens and can incur costs : Supports Speech Engines : @@ -222,71 +331,86 @@ Supports Speech Engines : * [Balcon](http://www.cross-plus-a.com/bconsole.htm) * Python [gtts](https://gtts.readthedocs.io/en/latest/) -``` +```python # choices "polly","balcon","gtts" voice_engine = "polly" ``` -Total number of reddit Videos to generate +### Limit the number of generated videos -``` +```python total_posts_to_process = 5 ``` The next settings are to automatically filter out posts +### Skip Reddit posts by submission score + Skip reddit posts that less than this amount of updates -``` +```python minimum_submission_score = 5000 ``` +### Filter Reddit posts by title length + Filtering out reddit posts based on the reddit post title length -``` +```python title_length_minimum = 20 title_length_maximum = 100 ``` +### Filter Reddit posts by self text length + Filter out posts that exceed the maximum self text length -``` +```python maximum_length_self_text = 5000 ``` +### Filter Reddit posts by comment count + Filter out reddit posts that don't have enough comments -``` +```python minimum_num_comments = 200 ``` +### Limit number of posts to process + Only attempt to process a maximum amount of reddit posts -``` +```python submission_limit = 1000 ``` +### Configure number of thumbnails to generate + Specify how many thumbnail images you want to generate -``` +```python number_of_thumbnails = 3 ``` +### Set the maximum video length Specify the maximum video length -``` +```python max_video_length = 600 # Seconds ``` +### Set maximum number of comments to include + Specify maximum amount of comments to generate in the video -``` +```python comment_limit = 600 ``` -Specifying various folder paths +### Specify folder paths -``` +```python assets_directory = "assets" temp_directory = "temp" audio_directory = str(Path("temp")) @@ -299,83 +423,91 @@ video_overlay_filepath = str(Path(assets_directory,"particles.mp4")) videos_directory = "videos" ``` -Specify video height and width +### Set video dimensions -``` +```python video_height = 720 video_width = 1280 clip_size = (video_width, video_height) ``` -Skip compiling the video and just exit instead +### Skip video compilation -``` +Skip compiling the video and just exit instead. + +```python enable_compilation = True ``` -Skip uploading to youtube +### Skip YouTube uploading -``` +```python enable_upload = False ``` +### Add a video overlay + Add a video overlay to the video, for example snow falling effect -``` +```python enable_overlay = True ``` +### Add a Newscaster + Add in a newscaster reader to the video -``` +```python enable_newscaster = True ``` -If newcaster video is a green screen attempt to remove the green screen +If the newcaster video is a green screen, attempt to remove the green screen -``` +```python newscaster_remove_greenscreen = True ``` Specify the color of the green screen in RGB -``` +```python newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour ``` -The higher the greenscreen threshold number the more it will attempt to remove +The higher the greenscreen threshold number the more it will attempt to remove. -``` +```python newscaster_greenscreen_remove_threshold = 100 ``` Path to newcaster file -``` +```python newscaster_filepath = str(Path(assets_directory,"newscaster.mp4").resolve()) ``` Position on the screen of the newscaster -``` +```python newscaster_position = ("left","bottom") ``` The size of the newscaster -``` +```python newcaster_size = (video_width * 0.5, video_height * 0.5) ``` +### Add a pause after each TTS file + Add a pause after each text to speech audio file -``` +```python pause = 1 # Pause after speech ``` -Text style settings +### Modify appearnace of text -``` +```python text_bg_color = "#1A1A1B" text_bg_opacity = 1 text_color = "white" @@ -383,8 +515,54 @@ text_font = "Verdana-Bold" text_fontsize = 32 ``` +### Download images from Lexica + Download images from lexica or skip trying to download -``` +```python lexica_download_enabled = True ``` +
+ +## Tips and Tricks + +### Generate a Video for a Specific Post + +or if you want to generate a video for a specific reddit post you can specify it via the `--url` param : + +```powershell +rybo --url https://www.reddit.com/r/AskReddit/comments/hvsxty/which_legendary_reddit_post_comment_can_you_still/ +``` + +or you can do multiple url's by seperating with a comma, ie : + +```powershell +rybo --url https://www.reddit.com/r/post1,https://www.reddit.com/r/post2,https://www.reddit.com/r/post3 +``` + +### Generate Only Thumbnails + +if you want to generate only thumbnails you can specify `--thumbnail-only` mode, this will skip video compilation process : + +```powershell +rybo --thumbnail-only +``` + +### Enable a Newscaster + +If you want to enable a Newscaster, edit settings.py and set : + +```python +enable_newscaster = True +``` + +![](assets/newscaster.png) + +If the newcaster video has a green screen you can remove it with the following settings, +use an eye dropper to get the RGB colour of the greenscreen and set it to have it removed : + +```python +newscaster_remove_greenscreen = True +newscaster_greenscreen_color = [1, 255, 17] # Enter the Green Screen RGB Colour +newscaster_greenscreen_remove_threshold = 100 +``` \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index ce19491..0000000 --- a/app.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Main entrypoint for the bot.""" -import logging -import os -import platform -import sys -from argparse import ArgumentParser, Namespace -from pathlib import Path -from typing import List - -from praw.models import Submission - -import config.settings as settings -import reddit.reddit as reddit -import thumbnail.thumbnail as thumbnail -import video_generation.video as vid -from csvmgr import CsvWriter -from utils.common import create_directory, safe_filename - -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) - - -class Video: - """Metadata used to splice content together to form a video.""" - - def __init__(self, submission): - """Initialize a Video instance. - - Args: - submission: The Reddit post to be converted into a video. - """ - self.submission = submission - self.comments = [] - self.clips = [] - self.background = None - self.music = None - self.thumbnail_path = None - self.folder_path = None - self.video_filepath = None - - -def process_submissions(submissions: List[Submission]) -> None: - """Prepare multiple reddit posts for conversion into YouTube videos. - - Args: - submissions: A list of zero or more Reddit posts to be converted. - """ - post_total: int = settings.total_posts_to_process - post_count: int = 0 - - for submission in submissions: - title_path: str = safe_filename(submission.title) - folder_path: Path = Path( - settings.videos_directory, f"{submission.id}_{title_path}" - ) - video_filepath: Path = Path(folder_path, "final.mp4") - if video_filepath.exists() or csvwriter.is_uploaded(submission.id): - print(f"Final video already processed : {submission.id}") - else: - process_submission(submission) - post_count += 1 - if post_count >= post_total: - print("Reached post count total!") - break - - -def process_submission(submission: Submission) -> None: - """Prepare a reddit post for conversion into a YouTube video. - - Args: - submission: The Reddit post to be converted into to a video. - """ - print("===== PROCESSING SUBMISSION =====") - print( - f"{str(submission.id)}, {str(submission.score)}, \ - {str(submission.num_comments)}, \ - {len(submission.selftext)}, \ - {submission.subreddit_name_prefixed}, \ - {submission.title}" - ) - video: Video = Video(submission) - title_path: str = safe_filename(submission.title) - - # Create Video Directories - video.folder_path: str = str( - Path(settings.videos_directory, f"{submission.id}_{title_path}") - ) - - create_directory(video.folder_path) - - video.video_filepath = str(Path(video.folder_path, "final.mp4")) - - if os.path.exists(video.video_filepath): - print(f"Final video already compiled : {video.video_filepath}") - else: - # Generate Thumbnail - - thumbnails: List[Path] = thumbnail.generate( - video_directory=video.folder_path, - subreddit=submission.subreddit_name_prefixed, - title=submission.title, - number_of_thumbnails=settings.number_of_thumbnails, - ) - - if thumbnails: - video.thumbnail_path = thumbnails[0] - - if args.thumbnail_only: - print("Generating Thumbnail only skipping video compile!") - else: - vid.create( - video_directory=video.folder_path, - post=submission, - thumbnails=thumbnails, - ) - - -def banner(): - """Display the CLIs banner.""" - print("##### YOUTUBE REDDIT BOT #####") - - -def print_version_info(): - """Display basic environment information.""" - print(f"OS Version : {platform.system()} {platform.release()}") - print(f"Python Version : {sys.version}") - - -def get_args() -> Namespace: - """Generate arguments supported by the CLI utility. - - Returns: - An argparse Namepsace containing the supported CLI parameters. - """ - parser: ArgumentParser = ArgumentParser() - - parser.add_argument( - "--enable-mentions", - action="store_true", - help="Check reddit account for u mentions", - ) - - parser.add_argument( - "--disable-selftext", - action="store_true", - help="Disable selftext video generation", - ) - - parser.add_argument( - "--voice-engine", - help="Specify which text to speech engine to use", - choices=["polly", "balcon", "gtts", "tiktok", "edge-tts", "streamlabspolly"], - ) - - parser.add_argument( - "-c", - "--comment-style", - help="Specify text based or reddit image comments", - choices=["text", "reddit"], - ) - - parser.add_argument( - "-l", "--video-length", help="Set how long you want the video to be", type=int - ) - - parser.add_argument( - "-n", "--enable-nsfw", action="store_true", help="Allow NSFW Content" - ) - - parser.add_argument( - "-o", "--disable-overlay", action="store_true", help="Disable video overlay" - ) - - parser.add_argument( - "-s", - "--story-mode", - action="store_true", - help="Generate video for post title and selftext only,\ - disables user comments", - ) - - parser.add_argument( - "-t", - "--thumbnail-only", - action="store_true", - help="Generate thumbnail image only", - ) - - parser.add_argument( - "-p", - "--enable-upload", - action="store_true", - help="Upload video to youtube, \ - requires client_secret.json and \ - credentials.storage to be valid", - ) - - parser.add_argument( - "-u", - "--url", - help="Specify Reddit post url, \ - seperate with a comma for multiple posts.", - ) - - parser.add_argument("--subreddits", help="Specify Subreddits, seperate with +") - - parser.add_argument( - "-b", - "--enable-background", - action="store_true", - help="Enable video backgrounds", - ) - - parser.add_argument("--total-posts", type=int, help="Enable video backgrounds") - - parser.add_argument( - "--submission-score", type=int, help="Minimum submission score threshold" - ) - - parser.add_argument( - "--background-directory", help="Folder path to video backgrounds" - ) - - parser.add_argument("--sort", choices=["top", "hot"], help="Sort Reddit posts by") - - parser.add_argument( - "--time", - choices=["all", "day", "hour", "month", "week", "year"], - default="day", - help="Filter by time", - ) - - parser.add_argument( - "--orientation", - choices=["landscape", "portrait"], - default="landscape", - help="Sort Reddit posts by", - ) - - parser.add_argument( - "--shorts", action="store_true", help="Generate Youtube Shorts Video" - ) - - args = parser.parse_args() - - if args.orientation: - settings.orientation = args.orientation - if args.orientation == "portrait": - settings.video_height = settings.vertical_video_height - settings.video_width = settings.vertical_video_width - logging.info("Setting Orientation to : %s", settings.orientation) - logging.info("Setting video_height to : %s", settings.video_height) - logging.info("Setting video_width to : %s", settings.video_width) - - if args.shorts: - logging.info("Generating Youtube Shorts Video") - settings.orientation = "portrait" - settings.video_height = settings.vertical_video_height - settings.video_width = settings.vertical_video_width - settings.max_video_length = 59 - settings.add_hashtag_shorts_to_description = True - - if args.enable_mentions: - settings.enable_reddit_mentions = True - logging.info("Enable Generate Videos from User Mentions") - - if args.submission_score: - settings.minimum_submission_score = args.submission_score - logging.info( - "Setting Reddit Post Minimum Submission Score : %s", - settings.minimum_submission_score, - ) - - if args.sort: - settings.reddit_post_sort = args.sort - logging.info("Setting Reddit Post Sort : %s", settings.reddit_post_sort) - if args.time: - settings.reddit_post_time_filter = args.time - logging.info( - "Setting Reddit Post Time Filter : %s", settings.reddit_post_time_filter - ) - - if args.background_directory: - logging.info( - "Setting video background directory : %s", args.background_directory - ) - settings.background_directory = args.background_directory - - if args.total_posts: - logging.info("Total Posts to process : %s", args.total_posts) - settings.total_posts_to_process = args.total_posts - - if args.comment_style: - logging.info("Setting comment style to : %s", args.comment_style) - settings.commentstyle = args.comment_style - - if args.voice_engine: - logging.info("Setting speech engine to : %s", args.voice_engine) - settings.voice_engine = args.voice_engine - - if args.video_length: - logging.info("Setting video length to : %s seconds", args.video_length) - settings.max_video_length = args.video_length - - if args.disable_overlay: - logging.info("Disabling Video Overlay") - settings.enable_overlay = False - - if args.enable_nsfw: - logging.info("Enable NSFW Content") - settings.enable_nsfw_content = True - - if args.story_mode: - logging.info("Story Mode Enabled!") - settings.enable_comments = False - - if args.disable_selftext: - logging.info("Disabled SelfText!") - settings.enable_selftext = False - - if args.enable_upload: - logging.info("Upload video enabled!") - settings.enable_upload = True - - if args.subreddits: - logging.info("Subreddits :") - settings.subreddits = args.subreddits.split("+") - print(settings.subreddits) - - if args.enable_background: - logging.info("Enabling Video Background!") - settings.enable_background = True - - return args - - -if __name__ == "__main__": - banner() - print_version_info() - args: Namespace = get_args() - csvwriter: CsvWriter = CsvWriter() - csvwriter.initialise_csv() - - submissions: List[Submission] = [] - - if args.url: - urls = args.url.split(",") - for url in urls: - submissions.append(reddit.get_reddit_submission(url)) - else: - if settings.enable_reddit_mentions: - logging.info("Getting Reddit Mentions") - mention_posts = reddit.get_reddit_mentions() - for mention_post in mention_posts: - logging.info("Reddit Mention : %s", mention_post) - submissions.append(reddit.get_reddit_submission(mention_post)) - - reddit_posts: List[Submission] = reddit.posts() - for reddit_post in reddit_posts: - submissions.append(reddit_post) - - submissions = reddit.get_valid_submissions(submissions) - - if submissions: - process_submissions(submissions) diff --git a/config/auth-env.py b/config/auth-env.py deleted file mode 100644 index 9d8499a..0000000 --- a/config/auth-env.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Load authentication credentials from environment variables.""" -import os - -print("Getting Secrets from ENV's") -# Reddit Praw -praw_client_id = os.environ.get("PRAW_CLIENT_ID") -praw_client_secret = os.environ.get("PRAW_CLIENT_SECRET") -praw_user_agent = os.environ.get("PRAW_USER_AGENT") -praw_password = os.environ.get("PRAW_PASSWORD") -praw_username = os.environ.get("PRAW_USERNAME") - -# Amazon Polly -aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID") -aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - -# Rumble -rumble_username = os.environ.get("RUMBLE_USERNAME") -rumble_password = os.environ.get("RUMBLE_PASSWORD") diff --git a/config/auth-example.py b/config/auth-example.py deleted file mode 100644 index cfe7fe7..0000000 --- a/config/auth-example.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Authentication credentials used by the bot.""" - -# FIXME: move credentials out of file. - -# Reddit Praw -praw_client_id = "xxxxxx" # noqa: S105 -praw_client_secret = "xxxxxx" # noqa: S105 -praw_user_agent = "xxxxxx" -praw_username = "xxxxxx" -praw_password = "xxxxxx" # noqa: S105 - -# Amazon Polly -aws_access_key_id = "xxxxxx" -aws_secret_access_key = "xxxxxx" # noqa: S105 - -# Rumble -rumble_username = "xxxxxx" -rumble_password = "xxxxxx" # noqa: S105 diff --git a/comments/cookie-dark-mode.json b/cookies/cookie-dark-mode.json similarity index 100% rename from comments/cookie-dark-mode.json rename to cookies/cookie-dark-mode.json diff --git a/comments/cookie-light-mode.json b/cookies/cookie-light-mode.json similarity index 100% rename from comments/cookie-light-mode.json rename to cookies/cookie-light-mode.json diff --git a/dependencies/scripts/postCreateCommand.sh b/dependencies/scripts/postCreateCommand.sh index f1c5dcc..c48380c 100644 --- a/dependencies/scripts/postCreateCommand.sh +++ b/dependencies/scripts/postCreateCommand.sh @@ -11,8 +11,14 @@ pip install --user -r requirements.txt # Additional dependencies needed to develop the utility & maintain code # quality. -pip install --user -r requirements-dev.txt +pip install --user -r requirements-dev.txt # Initialise playwright +playwright install playwright install-deps -playwright install firefox + +# Install an editable version of the rybo utility. Code changes will be applied +# in real-time. +pip install --editable . + +/bin/bash \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 50494a7..bf29102 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,47 @@ +[build-system] +requires = ["setuptools>=41", "wheel", "setuptools-git-versioning<2"] +build-backend = "setuptools.build_meta" + +[project] +name = "rybo" +dynamic = ["version"] +authors = [ + { name="Alex Laverty" } +] +maintainers = [ + { name="Alex Laverty" }, + { name="Nathon Fowlie" } +] +description="Reddit to YouTube bot." +readme="README.md" +requires-python=">=3.11" +classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent" +] +keywords = [ + "reddit", + "youtube", + "text-to-speech", + "bot" +] +license={file="LICENSE"} + +[project.urls] +"Homepage" = "https://github.com/alexlaverty/python-reddit-youtube-bot" +"Documentation" = "http://alexlaverty.github.io/python-reddit-youtube-bot" +"Repository" = "https://github.com/alexlaverty/python-reddit-youtube-bot.git" +"Bug Tracker" = "https://github.com/alexlaverty/python-reddit-youtube-bot/issues" + +[project.scripts] +rybo = "rybo.cli.rybo:cli" + [tool.flake8] -max-line-length = 88 +docstring-convention = 'google' +exclude = 'venv' extend-ignore = ['E203'] +max-line-length = 88 [tool.isort] profile = "black" @@ -11,3 +52,6 @@ max-line-length = 88 [tool.pycodestyle] max-line-length = 88 ignore = ['E203'] + +[tool.setuptools-git-versioning] +enabled = true \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 12ccdc6..cb8fe6d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8-bandit==4.1.1 flake8-black==0.3.6 flake8-bugbear==23.3.23 flake8-docstrings==1.7.0 -flake8-isort==4.0.0 +#flake8-isort==6.0.0 flake8-pyproject==1.2.3 flake8-secure-coding-standard==1.4.0 mypy==1.2.0 diff --git a/requirements.txt b/requirements.txt index dc3d538..6ee39c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ boto3==1.26.123 bs4==0.0.1 +colorama==0.4.6 +configparser==5.3.0 edge-tts==6.1.3 emoji==2.2.0 gtts==2.3.2 @@ -11,6 +13,7 @@ praw==7.7.0 pytime==0.2.3 pyyaml==6.0.0 rich==13.3.5 +ruamel.yaml==0.17.26 selenium==4.9.0 simple-youtube-api==0.2.8 varname==0.11.1 diff --git a/rybo.yaml b/rybo.yaml new file mode 100644 index 0000000..0a06954 --- /dev/null +++ b/rybo.yaml @@ -0,0 +1,82 @@ +--- + +# Folder path to videos that will be used for the video background. +background_directory: assets/backgrounds + +# Use text based, or image based Reddit comments. Choices are **text** or **image**. +comment_style: reddit + +# Enable or disable the video overlay. +disable_overlay: False + +# Enable or disable self-text video generation. +disable_selftext: False + +# Enable or disable adding a background to the video. +enable_background: False + +# Check the reddit account for user mentions. +enable_mentions: False + +# Include, or ignore posts tagged as Not Safe for Work. +enable_nsfw: False + +# Enable or disable uploading videos to YouTube. +enable_upload: False + +# Set the video orientation. Choices are "portrait" or "Landscape". +orientation: landscape + +# Polly credentials for the AWS API. +polly: + access_key_id: + secret_access_key: + +# Reddit credentials for the API and Web UI. +reddit: + client_id: + client_secret: + username: + password: + +# Rumble credentials for the Web UI. +rumble: + username: + password: + +# Enable or disable generating a YouTube shorts video. +shorts: False + +# Set the sorting order when scanning Reddit posts. Choices are "top" or "hot". +sort: hot + +# Enable or disable video generation for the post title and selftext only, +# disables user comments. +story_mode: False + +# Minimum submission score threshold. +submission_score: 5000 + +# List of subreddits to scan, where each subreddit is separated with a "+". +subreddits: antiwork+AskMen+askreddit+ChoosingBeggars+confession+confessions+hatemyjob+NoStupidQuestions+pettyrevenge+Showerthoughts+TooAfraidToAsk+TwoXChromosomes+unpopularopinion + +# Enable or disable generation of just the video thumbnails. +thumbnail_only: False + +# Filter Reddit submissions by time. Choices are "all", "day", "hour", +# "month", "week" or "year". +time: day + +# Total number of reddit submissions to process. +total_posts: 10 + +# Generate a video for a single Reddit submission. +url: + +# Sets how long the generated video will be, in seconds. +video_length: 600 + +# Specify which text-to-speech engine should be used to narrate the video. +# Choices are "polly", "balcon", "gtts", "tiktok", "edge-tts" or +# "streamlabspolly". +voice_engine: edge-tts diff --git a/src/rybo/__init__.py b/src/rybo/__init__.py new file mode 100644 index 0000000..d56549f --- /dev/null +++ b/src/rybo/__init__.py @@ -0,0 +1,5 @@ +"""Reddit to YouTube bot.""" +import importlib.metadata + +# Use pyproject.toml as the single source of truth for the package version. +__version__ = importlib.metadata.version(__name__) diff --git a/src/rybo/cli/__init__.py b/src/rybo/cli/__init__.py new file mode 100644 index 0000000..1184d0d --- /dev/null +++ b/src/rybo/cli/__init__.py @@ -0,0 +1 @@ +"""Rybo command line utility.""" diff --git a/src/rybo/cli/rybo.py b/src/rybo/cli/rybo.py new file mode 100644 index 0000000..bf8ce9f --- /dev/null +++ b/src/rybo/cli/rybo.py @@ -0,0 +1,525 @@ +"""Command line utility to generate YouTube videos from Reddit posts.""" + +import logging +import logging.config +import os +import platform +import sys +from argparse import ArgumentParser, Namespace +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List + +import colorama +from colorama import Fore +from ruamel.yaml import YAML + +from rybo import __version__ +from rybo.commands import RyboCommand +from rybo.config import settings +from rybo.logging import DEFAULT_LOG_CONFIG +from rybo.utils import EnvDefault, EnvFlagDefault + +# TODO: Placeholder for sub-commands. +# from rybo.commands.thumbnails import CreateThumbnailCommand +# from rybo.commands.video import CreateVideoCommand +# from rybo.commands.reddit import ExtractRedditCommand +# from rbo.commands.youtube import PublishYouTubeCommand + + +DEFAULT_CONFIG_FILE: Path = Path.joinpath(Path.home(), "rybo.yaml") +logger = logging.getLogger(__name__) + + +def cli() -> None: + """Create the rybo command line utility. + + Configuration options are loaded in the following order: + + 1. From a configuration file (default: $HOME/rybo.yaml). + 2. From environment variables that start with the prefix `RYBO_`. + 3. From user provided CLI parameters. + + Returns: + Returns an ArgumentParser that contains all of the sub-commands by rybo. + """ + _display_banner() + + # TODO: This needs refactoring + config_argparse: ArgumentParser = _configfile_parser() + config_args, _ = config_argparse.parse_known_args() + + defaults: Dict[str, Any] = DEFAULT_LOG_CONFIG + + if config_args.config is None: + config_args.config = DEFAULT_CONFIG_FILE + + cfg: Dict[str, Any] = _load_config(config_args) + cfg |= defaults + + # There are some additional environment variables that can be set to + # override configuration provided as CLI parameters, or via a configuration + # file, so need to load these last and write over the top of any values + # that already exist in the defaults dictionary. + envvars: Dict[str, Any] = _parse_env_overrides() + cfg |= envvars + + # Allows the user to change logging behaviour via a configuration file. + logging.config.dictConfig(cfg) + + parser: ArgumentParser = _cli(config_argparse, cfg) + _add_standard_args(parser) + + # TODO: placeholder for sub-commands + # subparser = parser.add_subparsers() + # _add_create_command(subparser) + + args: Namespace = parser.parse_args() + _log_configuration(args) + + print(args) + + command: Any = args.cmd + command.execute(args) + + +def _add_standard_args(parser: ArgumentParser) -> None: + """Common CLI command arguments. + + Args: + parser: Main parser that provides the rybo utility. + """ # noqa: D401 + # TODO: Needs refactoring + parser.add_argument( + "--version", + action="version", + version="%(prog)s {version}".format(version=__version__), + ) + parser.add_argument( + "--background-directory", + action=EnvFlagDefault, + envvar="RYBO_BACKGROUND_DIRECTORY", + help="Folder path to video backgrounds (env var: RYBO_BACKGROUND_DIRECORY).", + ) + parser.add_argument( + "-c", + "--comment-style", + action=EnvDefault, + envvar="RYBO_COMMENT_STYLE", + help="Specify text based or reddit image comments \ + (env var: RYBO_COMMENT_STYLE).", + choices=["text", "reddit"], + ) + parser.add_argument( + "-o", + "--disable-overlay", + action=EnvFlagDefault, + envvar="RYBO_DISABLE_OVERLAY", + help="Disable video overlay (env var: RYBO_DISABLE_OVERLAY).", + ) + parser.add_argument( + "--disable-selftext", + action=EnvFlagDefault, + envvar="RYBO_DISABLE_SELFTEXT", + help="Disable selftext video generation (env var: RYBO_DISABLE_SELFTEXT).", + ) + parser.add_argument( + "-b", + "--enable-background", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_BACKGROUND", + help="Enable video backgrounds (env var: RYBO_ENABLE_BACKGROUND).", + ) + parser.add_argument( + "--enable-mentions", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_MENTIONS", + help="Check reddit account for user mentions (env var: RYBO_ENABLE_MENTIONS).", + ) + parser.add_argument( + "-n", + "--enable-nsfw", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_NSFW", + help="Allow NSFW Content (env var: RYBO_ENABLE_NSFW).", + ) + parser.add_argument( + "-p", + "--enable-upload", + action=EnvFlagDefault, + envvar="RYBO_ENABLE_UPLOAD", + help="Upload video to youtube,requires client_secret.json and credentials.\ + storage to be valid (env var: RYBO_ENABLE_UPLOAD).", + ) + parser.add_argument( + "--orientation", + choices=["landscape", "portrait"], + action=EnvDefault, + envvar="RYBO_ORIENTATION", + default="landscape", + help="Sort Reddit posts by (env var: RYBO_ORIENTATION).", + ) + parser.add_argument( + "--shorts", + action=EnvFlagDefault, + envvar="RYBO_SHORTS", + help="Generate Youtube Shorts Video (env var: RYBO_SHORTS).", + ) + parser.add_argument( + "--sort", + action=EnvDefault, + envvar="RYBO_SORT", + choices=["top", "hot"], + help="Sort Reddit posts by (env var: RYBO_SORT).", + ) + parser.add_argument( + "--submission-score", + action=EnvDefault, + envvar="RYBO_SUBMISSION_SCORE", + type=int, + help="Minimum submission score threshold (env var: RYBO_SUBMISSION_SCORE).", + ) + parser.add_argument( + "--subreddits", + action=EnvDefault, + envvar="RYBO_SUBREDDITS", + help="Specify Subreddits, seperate with + (env var: RYBO_SUBREDDITS).", + ) + parser.add_argument( + "-s", + "--story-mode", + action=EnvFlagDefault, + envvar="RYBO_STORY_MODE", + help="Generate video for post title and selftext only, disables user comments \ + (env var: RYBO_STORY_MODE).", + ) + parser.add_argument( + "-t", + "--thumbnail-only", + action=EnvFlagDefault, + envvar="RYBO_THUMBNAIL_ONLY", + help="Generate thumbnail image only (env var: RYBO_THUMBNAIL_ONLY).", + ) + parser.add_argument( + "--time", + action=EnvDefault, + envvar="RYBO_TIME", + choices=["all", "day", "hour", "month", "week", "year"], + default="day", + help="Filter by time (env var: RYBO_TIME).", + ) + parser.add_argument( + "--total-posts", + action=EnvDefault, + envvar="RYBO_TOTAL_POSTS", + type=int, + help="Number of posts to process (env var: RYBO_TOTAL_POSTS).", + ) + parser.add_argument( + "-u", + "--url", + action=EnvDefault, + envvar="RYBO_URL", + help="Specify Reddit post url, seperate with a comma for multiple posts \ + (env var: RYBO_URL).", + ) + parser.add_argument( + "-l", + "--video-length", + action=EnvDefault, + envvar="RYBO_VIDEO_LENGTH", + type=int, + help="Set how long you want the video to be (env var: RYBO_VIDEO_LENGTH).", + ) + parser.add_argument( + "--voice-engine", + action=EnvDefault, + envvar="RYBO_VOICE_ENGINE", + choices=["polly", "balcon", "gtts", "tiktok", "edge-tts", "streamlabspolly"], + help="Specify which text to speech engine to use (env var: RYBO_VOICE_ENGINE).", + ) + + args = parser.parse_args() + + if args.orientation: + settings.orientation = args.orientation + if args.orientation == "portrait": + settings.video_height = settings.vertical_video_height + settings.video_width = settings.vertical_video_width + + if args.shorts: + logger.info("Generating Youtube Shorts Video") + settings.orientation = "portrait" + settings.video_height = settings.vertical_video_height + settings.video_width = settings.vertical_video_width + settings.max_video_length = 59 + settings.add_hashtag_shorts_to_description = True + + if args.enable_mentions: + settings.enable_reddit_mentions = True + + if args.submission_score: + settings.minimum_submission_score = args.submission_score + + if args.sort: + settings.reddit_post_sort = args.sort + + if args.time: + settings.reddit_post_time_filter = args.time + + if args.background_directory: + settings.background_directory = args.background_directory + + if args.total_posts: + settings.total_posts_to_process = args.total_posts + + if args.comment_style: + settings.commentstyle = args.comment_style + + if args.voice_engine: + settings.voice_engine = args.voice_engine + + if args.video_length: + settings.max_video_length = args.video_length + + if args.disable_overlay: + settings.enable_overlay = False + + if args.enable_nsfw: + settings.enable_nsfw_content = True + + if args.story_mode: + settings.enable_comments = False + + if args.disable_selftext: + settings.enable_selftext = False + + if args.enable_upload: + settings.enable_upload = True + + if args.subreddits: + settings.subreddits = args.subreddits.split("+") + logger.info("Subreddits :") + logger.info(settings.subreddits) + + if args.enable_background: + settings.enable_background = True + + parser.set_defaults(cmd=RyboCommand(parser)) + + +def _cli(parser: ArgumentParser, defaults: Dict[str, Any]) -> ArgumentParser: + """Create the main CLI parser. + + Where a configuration file is found, this will load CLI options from the + file, over-riding any configuration set via environment variables. + + Args: + parser: Parser used to read the configuration file. + defaults: Dictionary that contains the CLI argument default values. + + Returns: + The main rybo parser. + """ + parsers: List[ArgumentParser] = [parser] + parser: ArgumentParser = ArgumentParser( + description="Generate vidoes from reddit posts.", parents=parsers + ) + parser.set_defaults(**defaults) + return parser + + +def _configfile_parser() -> ArgumentParser: + """Add the CLI argument used to load a custom config file. + + Returns: + An `ArgumentParser` instance with the nargument used to specify the + location of a custom configuration file. + """ + parser: ArgumentParser = ArgumentParser(prog=__file__, add_help=False) + parser.add_argument( + "--config", + default=DEFAULT_CONFIG_FILE, + action=EnvDefault, + envvar="RYBO_CONFIG_FILE", + help="Path to the configuration file.", + ) + return parser + + +def _display_banner() -> None: + """Display the CLIs banner.""" + colorama.init(autoreset=True) + + python_version: str = f"{'Python Version':<20} : {sys.version}" + os_version: str = f"{'OS Version':<20} : {platform.system()} {platform.release()}" + rybo_version: str = f"{'Rybo Version':<20} : {__version__}" + title: str = "YOUTUBE REDDIT BOT" + + python_version_width: int = len(python_version) + os_version_width: int = len(os_version) + + if len(python_version) > len(os_version): + title = f"{' YOUTUBE REDDIT BOT ':=^{python_version_width}}" + else: + title = f"{' YOUTUBE REDDIT BOT ':=^{os_version_width}}" + + print(f"{Fore.CYAN}{title}\n{os_version}\n{python_version}\n{rybo_version}\n") + + +def _load_config(args: Namespace) -> Dict[str, Any]: + """Load and parser a configuration file. + + Args: + args: CLI arguments. + + Returns: + A dictionary containing configuration options. + """ + config: Dict[str, Any] = {} + config_file: Path = Path(args.config) + if config_file.is_file(): + yaml: YAML = YAML(typ="safe") + + # ruamel doesn't have the same vulnerability as the standard yaml + # library, so it's safe to ignore SCS105 here. + config = yaml.load(config_file) # noqa: SCS105 + + return config + + +def _log_configuration(args: Namespace) -> None: + """Log the bot configuration settings. + + Args: + args: ArgumentParser containing the user provided runtime parameters. + """ + if args.orientation: + logger.info(f"{'Set orientation to':<45} : %s", settings.orientation) + logger.info(f"{'Set video height to':<45} : %s", settings.video_height) + logger.info(f"{'Set video width to':<45} : %s", settings.video_width) + + if args.shorts: + logger.info("Generating Youtube Shorts Video") + + if args.enable_mentions: + logger.info("Enable Generate Videos from User Mentions") + + if args.submission_score: + logger.info( + f"{'Setting Reddit Post Minimum Submission Score':<45} : %s", + settings.minimum_submission_score, + ) + + if args.sort: + logger.info(f"{'Setting Reddit Post Sort':<45} : %s", settings.reddit_post_sort) + + if args.time: + logger.info( + f"{'Setting Reddit Post Time Filter':<45} : %s", + settings.reddit_post_time_filter, + ) + + if args.background_directory: + logger.info( + f"{'Setting video background directory':<45} : %s", + args.background_directory, + ) + + if args.total_posts: + logger.info(f"{'Total Posts to process':<45} : %s", args.total_posts) + + if args.comment_style: + logger.info(f"{'Setting comment style to':<45} : %s", args.comment_style) + + if args.voice_engine: + logger.info(f"{'Setting speech engine to':<45} : %s", args.voice_engine) + + if args.video_length: + logger.info(f"{'Setting video length to':<45} : %s seconds", args.video_length) + + if args.disable_overlay: + logger.info("Disabling Video Overlay") + + if args.enable_nsfw: + logger.info("Enable NSFW Content") + + if args.story_mode: + logger.info("Story Mode Enabled!") + + if args.disable_selftext: + logger.info("Disabled SelfText!") + + if args.enable_upload: + logger.info("Upload video enabled!") + + if args.subreddits: + logger.info("Subreddits :") + settings.subreddits = args.subreddits.split("+") + logger.info(settings.subreddits) + + if args.enable_background: + logger.info("Enabling Video Background!") + + +def _parse_env_overrides() -> Dict[str, Any]: + """Load rybo configuration provided as environment variables. + + Returns: + A dictionary containing the configuration specified as + environment variables. + """ + cfg: Dict[str, Any] = defaultdict(dict) + + if os.environ.get("RYBO_REDDIT_CLIENT_ID"): + cfg["reddit"]["client_id"] = os.environ["RYBO_REDDIT_CLIENT_ID"] + + if os.environ.get("RYBO_REDDIT_CLIENT_SECRET"): + cfg["reddit"]["client_secret"] = os.environ["RYBO_REDDIT_CLIENT_SECRET"] + + if os.environ.get("RYBO_REDDIT_USERNAME"): + cfg["reddit"]["username"] = os.environ["RYBO_REDDIT_USERNAME"] + + if os.environ.get("RYBO_REDDIT_PASSWORD"): + cfg["reddit"]["password"] = os.environ["RYBO_REDDIT_PASSWORD"] + + if os.environ.get("RYBO_POLLY_ACCESS_KEY"): + cfg["polly"]["access_key_id"] = os.environ["RYBO_POLLY_ACCESS_KEY"] + + if os.environ.get("RYBO_POLLY_SECRET_ACCESS_KEY"): + cfg["polly"]["secret_access_key"] = os.environ["RYBO_POLLY_SECRET_ACCESS_KEY"] + + if os.environ.get("RYBO_RUMBLE_USERNAME"): + cfg["rumble"]["username"] = os.environ["RYBO_RUMBLE_USERNAME"] + + if os.environ.get("RYBO_RUMBLE_PASSWORD"): + cfg["rumble"]["password"] = os.environ["RYBO_RUMBLE_PASSWORD"] + + return dict(cfg) + + +# TODO: Placeholder for sub-commands. +# def _add_create_command(subparser: _SubParsersAction): +# """Sub-command used to create new Zephyr folders. + +# Args: +# subparser: Parent that the sub-command will belong to. +# """ +# parser = subparser.add_parser('create', help='Create a new folder.') +# parser.add_argument( +# '--project', +# required=True, +# help='Project key of the project that the folder will be created under.' +# ) +# parser.add_argument( +# '--name', +# required=False, +# help='Name of the folder.' +# ) +# parser.add_argument( +# '--type', +# required=False, +# choices=['plan', 'case', 'cycle'], +# help='Type of folder to create.', +# ) +# parser.set_defaults(cmd=CreateFolderCommand(parser)) diff --git a/src/rybo/commands/__init__.py b/src/rybo/commands/__init__.py new file mode 100644 index 0000000..a833557 --- /dev/null +++ b/src/rybo/commands/__init__.py @@ -0,0 +1,229 @@ +"""Commands that can be executed by the rybo CLI utility.""" + +import logging +from abc import ABC, abstractmethod +from argparse import ArgumentParser, Namespace +from pathlib import Path +from typing import List + +from praw.models import Submission + +import rybo.config.settings as settings +import rybo.thumbnail.thumbnail as thumbnail +import rybo.video_generation.video as vid +from rybo.reddit import reddit +from rybo.utils.common import create_directory, safe_filename +from rybo.utils.csvmgr import CsvWriter + +logger = logging.getLogger(__name__) + + +class Video: + """Metadata used to splice content together to form a video.""" + + def __init__(self, submission): + """Initialize a Video instance. + + Args: + submission: The Reddit post to be converted into a video. + """ + self.submission = submission + self.comments = [] + self.clips = [] + self.background = None + self.music = None + self.thumbnail_path = None + self.folder_path = None + self.video_filepath = None + + +class CommandBase(ABC): + """Base class for CLI commands.""" + + def __init__(self, cli: ArgumentParser) -> None: + """Initialise the sub-command. + + Args: + cli: ArgParse action that will execute this command. + """ + self._cli = cli + self._logger = logging.getLogger(__name__) + + @abstractmethod + def execute(self, args: Namespace) -> None: + """Execute the command. + + Args: + args: User provided CLI arguments. + """ + pass + + +class RyboCommand(CommandBase): + """Top level command for the rybo utility. + + _See Also_: + [CommandBase][zfr.commands.CommandBase] + """ + + def execute(self, args: Namespace) -> None: + """Execute the command. + + This will simply display help information, as the ```FolderCommand``` + command acts as a wrapper for sub-commands used to create/update + Zephyr folders. + + Args: + args: User provided CLI arguments. + """ + csvwriter: CsvWriter = CsvWriter() + csvwriter.initialise_csv() + + submissions: List[Submission] = [] + + if args.url: + urls: List[str] = args.url.split(",") + for url in urls: + submissions.append( + reddit.get_reddit_submission( + url=url, + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + ) + else: + if settings.enable_reddit_mentions: + logger.info("Getting Reddit Mentions") + mention_posts = reddit.get_reddit_mentions( + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + for mention_post in mention_posts: + logger.info("Reddit Mention : %s", mention_post) + submissions.append( + reddit.get_reddit_submission( + url=mention_post, + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + ) + + reddit_posts: List[Submission] = reddit.posts( + client_id=args.reddit["client_id"], + client_secret=args.reddit["client_secret"], + ) + for reddit_post in reddit_posts: + submissions.append(reddit_post) + + submissions = reddit.get_valid_submissions(submissions) + + if submissions: + self._process_submissions( + submissions=submissions, + csvwriter=csvwriter, + thumbnail_only=args.thumbnail_only, + username=args.reddit["username"], + password=args.reddit["password"], + ) + + def _process_submissions( + self, + username: str, + password: str, + submissions: List[Submission], + csvwriter: CsvWriter, + thumbnail_only: bool = False, + ) -> None: + """Prepare multiple reddit posts for conversion into YouTube videos. + + Args: + submissions: A list of zero or more Reddit posts to be converted. + csvwriter: Helper object used to manage CSV files. + thumbnail_only: `True` to only generate a thumbnail and skip generating a + video, ottherwise `False` (default: False) + username: Reddit username. + password: Reddit password. + """ + post_total: int = settings.total_posts_to_process + post_count: int = 0 + + for submission in submissions: + title_path: str = safe_filename(submission.title) + folder_path: Path = Path( + settings.videos_directory, f"{submission.id}_{title_path}" + ) + video_filepath: Path = Path(folder_path, "final.mp4") + if video_filepath.exists() or csvwriter.is_uploaded(submission.id): + logger.info(f"Final video already processed : {submission.id}") + else: + self._process_submission( + submission=submission, + thumbnail_only=thumbnail_only, + username=username, + password=password, + ) + post_count += 1 + if post_count >= post_total: + logger.info("Reached post count total!") + break + + def _process_submission( + self, + username: str, + password: str, + submission: Submission, + thumbnail_only: bool = False, + ) -> None: + """Prepare a reddit post for conversion into a YouTube video. + + Args: + submission: The Reddit post to be converted into to a video. + thumbnail_only: `True` to only generate a thumbnail and skip generating a + video, ottherwise `False` (default: False) + username: Reddit username. + password: Reddit password. + """ + logger.info("===== PROCESSING SUBMISSION =====") + logger.info( + f"{str(submission.id)}, {str(submission.score)}, \ + {str(submission.num_comments)}, \ + {len(submission.selftext)}, \ + {submission.subreddit_name_prefixed}, \ + {submission.title}" + ) + video: Video = Video(submission) + title_path: str = safe_filename(submission.title) + + # Create Video Directories + video.folder_path: str = str( + Path(settings.videos_directory, f"{submission.id}_{title_path}") + ) + + create_directory(video.folder_path) + + video.video_filepath: Path = Path(video.folder_path, "final.mp4") + + if video.video_filepath.exists(): + logger.info(f"Final video already compiled : {video.video_filepath}") + else: + # Generate Thumbnail + thumbnails: List[Path] = thumbnail.generate( + video_directory=str(video.folder_path), + subreddit=submission.subreddit_name_prefixed, + title=submission.title, + number_of_thumbnails=settings.number_of_thumbnails, + ) + + if thumbnails: + video.thumbnail_path = thumbnails[0] + + if thumbnail_only: + logger.info("Generating Thumbnail only skipping video compile!") + else: + vid.create( + video_directory=video.folder_path, + post=submission, + thumbnails=thumbnails, + username=username, + password=password, + ) diff --git a/src/rybo/comments/__init__.py b/src/rybo/comments/__init__.py new file mode 100644 index 0000000..a314754 --- /dev/null +++ b/src/rybo/comments/__init__.py @@ -0,0 +1 @@ +"""Reddit comment processors.""" diff --git a/comments/screenshot.py b/src/rybo/comments/screenshot.py similarity index 81% rename from comments/screenshot.py rename to src/rybo/comments/screenshot.py index 6199c5c..c903e2a 100644 --- a/comments/screenshot.py +++ b/src/rybo/comments/screenshot.py @@ -1,5 +1,6 @@ """Take screenshots of Reddit comments.""" import json +import logging import os import re from io import TextIOWrapper @@ -10,27 +11,19 @@ from praw.models import Comment from rich.progress import track -import config.auth as auth -import config.settings as settings +from rybo.config import settings storymode = False - -def safe_filename(text: str): - """Replace spaces with an underscore. - - Args: - text: Filename to be sanitized. - - Returns: - A sanitized filename, where spaces have been replaced with underscores. - """ - text = text.replace(" ", "_") - return "".join([c for c in text if re.match(r"\w", c)])[:50] +logger = logging.getLogger(__name__) def download_screenshots_of_reddit_posts( - accepted_comments: List[Comment], url: str, video_directory: Path + accepted_comments: List[Comment], + url: str, + video_directory: Path, + username: str, + password: str, ) -> None: """Download screenshots of reddit posts as seen on the web. @@ -40,8 +33,10 @@ def download_screenshots_of_reddit_posts( accepted_comments: List of comments to be included in the video. url: URL of the Reddit content to be screenshotted. video_directory: Path where the screenshots will be saved. + username: Reddit username. + password: Reddit password. """ - print("Downloading screenshots of reddit posts...") + logger.info("Downloading screenshots of reddit posts...") # id = re.sub(r"[^\w\s-]", "", reddit_object.meta.id) # # ! Make sure the reddit screenshots folder exists # title_path = safe_filename(reddit_object.title) @@ -49,26 +44,26 @@ def download_screenshots_of_reddit_posts( # #Path(f"assets/temp/{id}/png").mkdir(parents=True, exist_ok=True) with sync_playwright() as p: - print("Launching Headless Browser...") + logger.debug("Launching Headless Browser...") browser = p.chromium.launch(headless=True) context = browser.new_context() context.set_default_timeout(settings.comment_screenshot_timeout) if settings.theme == "dark": cookie_file: TextIOWrapper = open( - f"{os.getcwd()}/comments/cookie-dark-mode.json", encoding="utf-8" + f"{os.getcwd()}/cookies/cookie-dark-mode.json", encoding="utf-8" ) else: cookie_file: TextIOWrapper = open( - f"{os.getcwd()}/comments/cookie-light-mode.json", encoding="utf-8" + f"{os.getcwd()}/cookies/cookie-light-mode.json", encoding="utf-8" ) # Get the thread screenshot page = context.new_page() page.goto("https://www.reddit.com/login") - page.type("#loginUsername", auth.praw_username) - page.type("#loginPassword", auth.praw_password) + page.type("#loginUsername", username) + page.type("#loginPassword", password) page.click('button[type="submit"]') page.wait_for_url("https://www.reddit.com/") @@ -81,7 +76,7 @@ def download_screenshots_of_reddit_posts( if page.locator('[data-testid="content-gate"]').is_visible(): # This means the post is NSFW and requires to click the proceed button. - print("Post is NSFW. You are spicy...") + logger.info("Post is NSFW. You are spicy...") page.locator('[data-testid="content-gate"] button').click() page.wait_for_load_state() # Wait for page to fully load @@ -101,7 +96,9 @@ def download_screenshots_of_reddit_posts( comment_path: Path = Path(f"{video_directory}/comment_{comment.id}.png") if comment_path.exists(): - print(f"Comment Screenshot already downloaded : {comment_path}") + logger.info( + f"Comment Screenshot already downloaded : {comment_path}" + ) else: if page.locator('[data-testid="content-gate"]').is_visible(): page.locator('[data-testid="content-gate"] button').click() @@ -113,7 +110,7 @@ def download_screenshots_of_reddit_posts( except Exception: # noqa: S110 pass - print("Screenshots downloaded Successfully.") + logger.info("Screenshots downloaded Successfully.") def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> None: @@ -125,10 +122,10 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> url: URL to take a screenshot of. video_directory: Path to save the screenshots to. """ - print("Downloading screenshots of reddit title...") - print(url) + logger.info("Downloading screenshots of reddit title...") + logger.info(url) with sync_playwright() as p: - print("Launching Headless Browser...") + logger.info("Launching Headless Browser...") browser = p.chromium.launch(headless=True) context = browser.new_context() @@ -149,7 +146,7 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> if page.locator('[data-testid="content-gate"]').is_visible(): # This means the post is NSFW and requires to click the proceed button. - print("Post is NSFW. You are spicy...") + logger.info("Post is NSFW. You are spicy...") page.locator('[data-testid="content-gate"] button').click() page.wait_for_load_state() # Wait for page to fully load @@ -166,4 +163,17 @@ def download_screenshot_of_reddit_post_title(url: str, video_directory: Path) -> path=f"{video_directory}/title.png" ) - print("Title Screenshot downloaded Successfully.") + logger.info("Title Screenshot downloaded Successfully.") + + +def safe_filename(text: str) -> str: + """Replace spaces with an underscore. + + Args: + text: Filename to be sanitized. + + Returns: + A sanitized filename, where spaces have been replaced with underscores. + """ + text = text.replace(" ", "_") + return "".join([c for c in text if re.match(r"\w", c)])[:50] diff --git a/src/rybo/config/__init__.py b/src/rybo/config/__init__.py new file mode 100644 index 0000000..0eed9d6 --- /dev/null +++ b/src/rybo/config/__init__.py @@ -0,0 +1 @@ +"""Rybo configuration.""" diff --git a/config/settings.py b/src/rybo/config/settings.py similarity index 100% rename from config/settings.py rename to src/rybo/config/settings.py index 97d43cc..011f089 100644 --- a/config/settings.py +++ b/src/rybo/config/settings.py @@ -1,6 +1,6 @@ """Configuration settings for the bot.""" -from sys import platform from pathlib import Path +from sys import platform subreddits = [ "askreddit", diff --git a/src/rybo/logging.py b/src/rybo/logging.py new file mode 100644 index 0000000..bf5c8a5 --- /dev/null +++ b/src/rybo/logging.py @@ -0,0 +1,30 @@ +"""Default logging configuration, and associated utilities.""" + +from typing import Any, Dict + +DEFAULT_LOG_CONFIG: Dict[str, Any] = { + "version": 1, + "disable_existing_loggers": False, + "propogate": False, + "formatters": { + "fmt": { + "format": "%(asctime)s %(levelname)-8s %(message)s", + "datefmt": "%d-%m-%Y %H:%M:%S", + } + }, + "handlers": { + "fh": { + "class": "logging.handlers.RotatingFileHandler", + "filename": "rybo.log", + "maxBytes": 1048576, + "backupCount": 3, + "formatter": "fmt", + }, + "sh": { + "class": "logging.StreamHandler", + "formatter": "fmt", + "stream": "ext://sys.stdout", + }, + }, + "loggers": {"rybo": {"handlers": ["fh", "sh"], "level": "DEBUG"}}, +} diff --git a/src/rybo/publish/__init__.py b/src/rybo/publish/__init__.py new file mode 100644 index 0000000..22150c5 --- /dev/null +++ b/src/rybo/publish/__init__.py @@ -0,0 +1 @@ +"""Video publishers.""" diff --git a/publish/login.py b/src/rybo/publish/login.py similarity index 94% rename from publish/login.py rename to src/rybo/publish/login.py index 7fb9d43..0e4d663 100644 --- a/publish/login.py +++ b/src/rybo/publish/login.py @@ -1,5 +1,6 @@ """Login module.""" import json +import logging from typing import Dict, List from selenium.webdriver.common.by import By @@ -7,6 +8,28 @@ from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait +logger = logging.getLogger(__name__) + + +def confirm_logged_in(driver: WebDriver) -> bool: + """Confirm that the user is logged in. + + The browser needs to be navigated to a YouTube page. + + Args: + driver: Selenium webdrive. + + Returns: + `True` if the user is logged in, otherwise `False`. + """ + try: + WebDriverWait(driver, 5).until( + ec.element_to_be_clickable((By.ID, "avatar-btn")) + ) + return True + except TimeoutError: + return False + def domain_to_url(domain: str) -> str: """Convert a partial domain to valid URL. @@ -55,24 +78,4 @@ def login_using_cookie_file(driver: WebDriver, cookie_file: str) -> None: try: driver.add_cookie(cookie) except Exception: - print(f"Couldn't set cookie {cookie['name']} for {domain}") - - -def confirm_logged_in(driver: WebDriver) -> bool: - """Confirm that the user is logged in. - - The browser needs to be navigated to a YouTube page. - - Args: - driver: Selenium webdrive. - - Returns: - `True` if the user is logged in, otherwise `False`. - """ - try: - WebDriverWait(driver, 5).until( - ec.element_to_be_clickable((By.ID, "avatar-btn")) - ) - return True - except TimeoutError: - return False + logger.info(f"Couldn't set cookie {cookie['name']} for {domain}") diff --git a/publish/upload.py b/src/rybo/publish/upload.py similarity index 90% rename from publish/upload.py rename to src/rybo/publish/upload.py index 8aacb76..be3fd69 100644 --- a/publish/upload.py +++ b/src/rybo/publish/upload.py @@ -2,8 +2,11 @@ import logging import re from datetime import datetime, timedelta +from pathlib import Path from time import sleep +import colorama +from colorama import Fore from selenium.common.exceptions import ( ElementNotInteractableException, NoSuchElementException, @@ -15,107 +18,55 @@ from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait +logger = logging.getLogger(__name__) today = datetime.now() + timedelta(minutes=15) +colorama.init(autoreset=True) -def upload_file( - driver: WebDriver, - video_path: str, - title: str, - description: str, - game: str = False, - kids: bool = False, - upload_time: datetime = None, - thumbnail_path: str = None, + +def _set_advanced_settings( + driver: WebDriver, game_title: str, made_for_kids: bool ) -> None: - """Upload a single video to YouTube. + """Associate the video with a game, and/or flag it as suitable for kids. Args: driver: Selenium webdriver. - video_path: Path to the video to be uploaded. - title: Video title as it will appear on YouTube. - description: Video description as it will appear on YouTube. - game: Game that the video is associated with. - kids: `True` to indicate that the video is suitable for viewing by - kids, otherwise `false`. - upload_time: Date and time that the video was uploaded. + game_title: Name of the game that the video is relevant to. + made_for_kids: `True` to indicate the video is suitable for viewing by + kids, otherwise `False`. """ - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable((By.CSS_SELecTOR, "ytcp-button#create-icon")) - ).click() - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - (By.XPATH, '//tp-yt-paper-item[@test-id="upload-beta"]') + logger.info("Setting Advanced Settings") + # Open advanced options + driver.find_element(By.CSS_SELecTOR, "#toggle-button").click() + if game_title: + game_title_input: WebElement = driver.find_element( + By.CSS_SELecTOR, + ".ytcp-form-gaming > " + "ytcp-dropdown-trigger:nth-child(1) > " + ":nth-child(2) > div:nth-child(3) > input:nth-child(3)", ) - ).click() - video_input = driver.find_element(by=By.XPATH, value='//input[@type="file"]') - logging.info("Setting Video File Path") - video_input.send_keys(video_path) + game_title_input.send_keys(game_title) - _set_basic_settings(driver, title, description, thumbnail_path) - _set_advanced_settings(driver, game, kids) - # Go to visibility settings - for _i in range(3): + # Select first item in game drop down WebDriverWait(driver, 20).until( - ec.element_to_be_clickable((By.ID, "next-button")) + ec.element_to_be_clickable( + ( + By.CSS_SELecTOR, + "#text-item-2", # The first item is an empty item + ) + ) ).click() - # _set_time(driver, upload_time) - try: - _set_visibility(driver) - except Exception: - print("error uploading, continuing") - _wait_for_processing(driver) - # Go back to endcard settings - # find_element(By.CSS_SELecTOR,"#step-badge-1").click() - # _set_endcard(driver) - - # for _ in range(2): - # # Sometimes, the button is clickable but clicking it raises an - # # error, so we add a "safety-sleep" here - # sleep(5) - # WebDriverWait(driver, 20) - # .until(ec.element_to_be_clickable((By.ID, "next-button"))) - # .click() - - # sleep(5) - # WebDriverWait(driver, 20) - # .until(ec.element_to_be_clickable((By.ID, "done-button"))) - # .click() - - # # Wait for the dialog to disappear - # sleep(5) - driver.close() - logging.info("Upload is complete") - - -def _wait_for_processing(driver: WebDriver) -> None: - """Wait for YouTube to process the video. - - Calling this method will cause progress updates to be sent to stdout - every 5 seconds until the processing is complete. - - Args: - driver: Selenium webdriver. - """ - logging.info("Waiting for processing to complete") - # Wait for processing to complete - progress_label: WebElement = driver.find_element( - By.CSS_SELecTOR, "span.progress-label" - ) - pattern = re.compile(r"(finished processing)|(processing hd.*)|(check.*)") - current_progress = progress_label.get_attribute("textContent") - last_progress = None - while not pattern.match(current_progress.lower()): - if last_progress != current_progress: - logging.info(f"Current progress: {current_progress}") - last_progress = current_progress - sleep(5) - current_progress = progress_label.get_attribute("textContent") - if "Processing 99" in current_progress: - print("Finished Processing!") - sleep(10) - break + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable( + ( + By.NAME, + "VIDEO_MADE_FOR_KIDS_MFK" + if made_for_kids + else "VIDEO_MADE_FOR_KIDS_NOT_MFK", + ) + ) + ).click() def _set_basic_settings( @@ -133,7 +84,7 @@ def _set_basic_settings( thumbnail_path: Path to be used to set the videos thumbnail, as it appears in YouTube search results. """ - logging.info("Setting Basic Settings") + logger.info("Setting Basic Settings") title_input: WebElement = WebDriverWait(driver, 20).until( ec.element_to_be_clickable( ( @@ -157,67 +108,22 @@ def _set_basic_settings( ) title_input.clear() - logging.info("Setting Video Title") + logger.info("Setting Video Title") title_input.send_keys(title) - logging.info("Setting Video Description") + logger.info("Setting Video Description") description_input.send_keys(description) if thumbnail_path: - logging.info("Setting Video Thumbnail") + logger.info("Setting Video Thumbnail") thumbnail_input.send_keys(thumbnail_path) -def _set_advanced_settings( - driver: WebDriver, game_title: str, made_for_kids: bool -) -> None: - """Associate the video with a game, and/or flag it as suitable for kids. - - Args: - driver: Selenium webdriver. - game_title: Name of the game that the video is relevant to. - made_for_kids: `True` to indicate the video is suitable for viewing by - kids, otherwise `False`. - """ - logging.info("Setting Advanced Settings") - # Open advanced options - driver.find_element(By.CSS_SELecTOR, "#toggle-button").click() - if game_title: - game_title_input: WebElement = driver.find_element( - By.CSS_SELecTOR, - ".ytcp-form-gaming > " - "ytcp-dropdown-trigger:nth-child(1) > " - ":nth-child(2) > div:nth-child(3) > input:nth-child(3)", - ) - game_title_input.send_keys(game_title) - - # Select first item in game drop down - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - ( - By.CSS_SELecTOR, - "#text-item-2", # The first item is an empty item - ) - ) - ).click() - - WebDriverWait(driver, 20).until( - ec.element_to_be_clickable( - ( - By.NAME, - "VIDEO_MADE_FOR_KIDS_MFK" - if made_for_kids - else "VIDEO_MADE_FOR_KIDS_NOT_MFK", - ) - ) - ).click() - - def _set_endcard(driver: WebDriver) -> None: """Set the end card. Args: driver: Selenium webdriver. """ - logging.info("Endscreen") + logger.info("Endscreen") # Add endscreen driver.find_element(By.CSS_SELecTOR, "#endscreens-button").click() @@ -229,7 +135,9 @@ def _set_endcard(driver: WebDriver) -> None: driver.find_element(By.CSS_SELecTOR, "div.card:nth-child(1)").click() break except (NoSuchElementException, ElementNotInteractableException): - logging.warning(f"Couldn't find endcard button. Retry in 5s! ({i}/10)") + logger.warning( + f"{Fore.YELLOW}Couldn't find endcard button. Retry in 5s! ({i}/10)" + ) sleep(5) WebDriverWait(driver, 20).until( @@ -284,7 +192,7 @@ def _set_visibility(driver: WebDriver) -> None: driver: Selenium webdriver. """ # Start time scheduling - logging.info("Setting Visibility to public") + logger.info("Setting Visibility to public") WebDriverWait(driver, 30).until( ec.element_to_be_clickable((By.NAME, "FIRST_CONTAINER")) ).click() @@ -298,3 +206,104 @@ def _set_visibility(driver: WebDriver) -> None: # sleep(10) # WebDriverWait(driver, 30) # .until(ec.element_to_be_clickable((By.ID, "close-button"))).click() + + +def upload_file( + driver: WebDriver, + video_path: Path, + title: str, + description: str, + game: str = False, + kids: bool = False, + upload_time: datetime = None, + thumbnail_path: str = None, +) -> None: + """Upload a single video to YouTube. + + Args: + driver: Selenium webdriver. + video_path: Path to the video to be uploaded. + title: Video title as it will appear on YouTube. + description: Video description as it will appear on YouTube. + game: Game that the video is associated with. + kids: `True` to indicate that the video is suitable for viewing by + kids, otherwise `false`. + upload_time: Date and time that the video was uploaded. + thumbnail_path: Path to the thumbnail to be used. + """ + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable((By.CSS_SELecTOR, "ytcp-button#create-icon")) + ).click() + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable( + (By.XPATH, '//tp-yt-paper-item[@test-id="upload-beta"]') + ) + ).click() + video_input = driver.find_element(by=By.XPATH, value='//input[@type="file"]') + logger.info("Setting Video File Path") + video_input.send_keys(video_path) + + _set_basic_settings(driver, title, description, thumbnail_path) + _set_advanced_settings(driver, game, kids) + # Go to visibility settings + for _i in range(3): + WebDriverWait(driver, 20).until( + ec.element_to_be_clickable((By.ID, "next-button")) + ).click() + + # _set_time(driver, upload_time) + try: + _set_visibility(driver) + except Exception: + logger.info("error uploading, continuing") + _wait_for_processing(driver) + # Go back to endcard settings + # find_element(By.CSS_SELecTOR,"#step-badge-1").click() + # _set_endcard(driver) + + # for _ in range(2): + # # Sometimes, the button is clickable but clicking it raises an + # # error, so we add a "safety-sleep" here + # sleep(5) + # WebDriverWait(driver, 20) + # .until(ec.element_to_be_clickable((By.ID, "next-button"))) + # .click() + + # sleep(5) + # WebDriverWait(driver, 20) + # .until(ec.element_to_be_clickable((By.ID, "done-button"))) + # .click() + + # # Wait for the dialog to disappear + # sleep(5) + driver.close() + logger.info("Upload is complete") + + +def _wait_for_processing(driver: WebDriver) -> None: + """Wait for YouTube to process the video. + + Calling this method will cause progress updates to be sent to stdout + every 5 seconds until the processing is complete. + + Args: + driver: Selenium webdriver. + """ + logger.info("Waiting for processing to complete") + # Wait for processing to complete + progress_label: WebElement = driver.find_element( + By.CSS_SELecTOR, "span.progress-label" + ) + pattern = re.compile(r"(finished processing)|(processing hd.*)|(check.*)") + current_progress = progress_label.get_attribute("textContent") + last_progress = None + while not pattern.match(current_progress.lower()): + if last_progress != current_progress: + logger.info(f"Current progress: {current_progress}") + last_progress = current_progress + sleep(5) + current_progress = progress_label.get_attribute("textContent") + if "Processing 99" in current_progress: + logger.info("Finished Processing!") + sleep(10) + break diff --git a/publish/youtube.py b/src/rybo/publish/youtube.py similarity index 78% rename from publish/youtube.py rename to src/rybo/publish/youtube.py index 7ef2e99..40b6d39 100644 --- a/publish/youtube.py +++ b/src/rybo/publish/youtube.py @@ -6,16 +6,13 @@ from simple_youtube_api.Channel import Channel from simple_youtube_api.LocalVideo import LocalVideo -#from simple_youtube_api.youtube_video import YouTubeVideo -import config.settings as settings +from rybo.config import settings -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log"), logging.StreamHandler()], -) +# from simple_youtube_api.youtube_video import YouTubeVideo + + +logger = logging.getLogger(__name__) @dataclass @@ -34,10 +31,10 @@ def publish(video: Video) -> None: Args: video: Metadata describing the video to be uploaded. """ - logging.info("========== Uploading Video To Youtube ==========") - logging.info("video.filepath : %s", video.filepath) - logging.info("video.title : %s", video.title) - logging.info("video.thumbnail : %s", video.thumbnail) + logger.info("========== Uploading Video To Youtube ==========") + logger.info("video.filepath : %s", video.filepath) + logger.info("video.title : %s", video.title) + logger.info("video.thumbnail : %s", video.thumbnail) # loggin into the channel channel: Channel = Channel() @@ -65,11 +62,11 @@ def publish(video: Video) -> None: try: # uploading video and printing the results uploaded_video = channel.upload_video(youtube_upload) - print(uploaded_video.id) - print(uploaded_video) + logger.info(uploaded_video.id) + logger.info(uploaded_video) except Exception as e: - logging.info("Error uploading video : %s", video.title) - print(e) + logger.info("Error uploading video : %s", video.title) + logger.info(e) if __name__ == "__main__": diff --git a/src/rybo/reddit/__init__.py b/src/rybo/reddit/__init__.py new file mode 100644 index 0000000..972def6 --- /dev/null +++ b/src/rybo/reddit/__init__.py @@ -0,0 +1 @@ +"""Reddit post processors.""" diff --git a/reddit/reddit.py b/src/rybo/reddit/reddit.py similarity index 65% rename from reddit/reddit.py rename to src/rybo/reddit/reddit.py index 2de57a3..3b03eda 100644 --- a/reddit/reddit.py +++ b/src/rybo/reddit/reddit.py @@ -1,157 +1,90 @@ """Helpers used to retreive, filter and process reddit posts.""" -import praw -import config.settings as settings -import config.auth as auth import base64 +import logging import re -from praw.models import Submission from typing import List +import praw +from praw.models import Submission -def is_valid_submission(submission) -> bool: - """Determine whether a Reddit post is worth turning in to a video. +from rybo.config import settings - A post is deemed "worthy", if: - - It isn't stickied. - - It was submitted by ttvibe. - - The posts title is within the min/max ranges defined in settings. - - The post hasn't been flagged as NSFW. - - The post doesn't contain banned keywords. - - The post wasn't made in a subreddit that has been added to the - ignore list. - - The submission score is within range. - - The length of the post content is less than the configured maximum - length. - - The post has more than a minimum number of comments. - - The post is not an update on a previous post. - - The post doesn't contain 'covid' or 'vaccine' in the title, as these - tend to trigger Youtube strikes. Censorship is double-plus good... +logger = logging.getLogger(__name__) + + +def get_reddit_mentions(client_id: str, client_secret: str) -> List[str]: + """Get a list of comments where ttvibe has been mentioned. Args: - submission: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. Returns: - `True` if the Reddit post is deemed worthy of turning in to a video, - otherwise returns `False`. + A list containing zero or more URLs where ttvibe has been mentioned. """ - if submission.stickied: - return False - - if not settings.enable_screenshot_title_image and not submission.is_self: - return False - - if ( - len(submission.title) < settings.title_length_minimum - or len(submission.title) > settings.title_length_maximum - ): - return False - - if not settings.enable_nsfw_content: - if submission.over_18: - print("Skipping NSFW...") - return False - for banned_keyword in ( - base64.b64decode(settings.banned_keywords_base64.encode("ascii")) - .decode("ascii") - .split(",") - ): - if banned_keyword in submission.title.lower(): - print( - f"{submission.title} \ - <-- Skipping post, title contains banned keyword!" - ) - return False - - if submission.subreddit_name_prefixed in settings.subreddits_excluded: - return False - - if submission.score < settings.minimum_submission_score: - print(f"{submission.title} <-- Submission score too low!") - return False - - if len(submission.selftext) > settings.maximum_length_self_text: - return False - - if ( - settings.enable_comments - and submission.num_comments < settings.minimum_num_comments - ): - print(f"{submission.title} <-- Number of comments too low!") - return False - - if "update" in submission.title.lower(): - return False + r: praw.Reddit = praw.Reddit( + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 \ + Firefox/112.0", + ) - if "covid" in submission.title.lower() or "vaccine" in submission.title.lower(): - print(f"{submission.title} <-- Youtube Channel Strikes if Covid content...!") - return False + mention_urls: List[str] = [] + for mention in r.inbox.mentions(limit=None): + post_url: str = re.sub( + rf"/{mention.id}/\?context=\d", "", mention.context, flags=re.IGNORECASE + ) + mention_urls.append(f"https://www.reddit.com{post_url}") - return True + return mention_urls -def get_reddit_submission(url: str) -> Submission: +def get_reddit_submission(url: str, client_id: str, client_secret: str) -> Submission: """Get a single Reddit post. Args: url: URL to the post to be retrieved. + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. Returns: The post contents. """ r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 \ + Firefox/112.0", ) submission: Submission = r.submission(url=url) return submission -def get_reddit_mentions() -> List[str]: - """Get a list of comments where ttvibe has been mentioned. - - Returns: - A list containing zero or more URLs where ttvibe has been mentioned. - """ - r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, - username=auth.praw_username, - password=auth.praw_password, - ) - - mention_urls: List[str] = [] - for mention in r.inbox.mentions(limit=None): - post_url: str = re.sub( - rf"/{mention.id}/\?context=\d", "", mention.context, flags=re.IGNORECASE - ) - mention_urls.append(f"https://www.reddit.com{post_url}") - - return mention_urls - - -def get_reddit_submissions() -> List[Submission]: +def get_reddit_submissions(client_id: str, client_secret: str) -> List[Submission]: """Get the latest Reddit posts. Posts will be retrieved according to the sort order defined in settings. + Args: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. + Returns: A list containing zero or more Reddit posts. """ r: praw.Reddit = praw.Reddit( - client_id=auth.praw_client_id, - client_secret=auth.praw_client_secret, - user_agent=auth.praw_user_agent, + client_id=client_id, + client_secret=client_secret, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101\ + Firefox/112.0", ) if settings.subreddits: subreddits: str = "+".join(settings.subreddits) else: subreddits: str = "all" - print("Retrieving posts from subreddit :") - print(subreddits) + logger.info("Retrieving posts from subreddit %s", subreddits) # Get Reddit Hot Posts if settings.reddit_post_sort == "hot": @@ -182,29 +115,120 @@ def get_valid_submissions(submissions: List[Submission]) -> List[Submission]: material, to generate a video. """ valid_submissions: List[Submission] = [] - print("===== Retrieving valid Reddit submissions =====") - print("ID, SCORE, NUM_COMMENTS, LEN_SELFTEXT, SUBREDDIT, TITLE") + logger.info("===== Retrieving valid Reddit submissions =====") + logger.info("ID, SCORE, NUM_COMMENTS, LEN_SELFTEXT, SUBREDDIT, TITLE") for submission in submissions: if is_valid_submission(submission): msg: str = ", ".join( - [str(submission.id), - str(submission.score), - str(submission.num_comments), - str(len(submission.selftext)), - submission.subreddit_name_prefixed, - submission.title] + [ + str(submission.id), + str(submission.score), + str(submission.num_comments), + str(len(submission.selftext)), + submission.subreddit_name_prefixed, + submission.title, + ] ) - print(msg) + logger.info(msg) valid_submissions.append(submission) return valid_submissions -def posts() -> List[Submission]: +def is_valid_submission(submission: Submission) -> bool: + """Determine whether a Reddit post is worth turning in to a video. + + A post is deemed "worthy", if: + - It isn't stickied. + - It was submitted by ttvibe. + - The posts title is within the min/max ranges defined in settings. + - The post hasn't been flagged as NSFW. + - The post doesn't contain banned keywords. + - The post wasn't made in a subreddit that has been added to the + ignore list. + - The submission score is within range. + - The length of the post content is less than the configured maximum + length. + - The post has more than a minimum number of comments. + - The post is not an update on a previous post. + - The post doesn't contain 'covid' or 'vaccine' in the title, as these + tend to trigger Youtube strikes. Censorship is double-plus good... + + Args: + submission: A single Reddit post. + + Returns: + `True` if the Reddit post is deemed worthy of turning in to a video, + otherwise returns `False`. + """ + if submission.stickied: + return False + + if not settings.enable_screenshot_title_image and not submission.is_self: + return False + + if ( + len(submission.title) < settings.title_length_minimum + or len(submission.title) > settings.title_length_maximum + ): + return False + + if not settings.enable_nsfw_content: + if submission.over_18: + logger.info("Skipping NSFW...") + return False + for banned_keyword in ( + base64.b64decode(settings.banned_keywords_base64.encode("ascii")) + .decode("ascii") + .split(",") + ): + if banned_keyword in submission.title.lower(): + logger.info( + f"{submission.title} \ + <-- Skipping post, title contains banned keyword!" + ) + return False + + if submission.subreddit_name_prefixed in settings.subreddits_excluded: + return False + + if submission.score < settings.minimum_submission_score: + logger.info(f"{submission.title} <-- Submission score too low!") + return False + + if len(submission.selftext) > settings.maximum_length_self_text: + return False + + if ( + settings.enable_comments + and submission.num_comments < settings.minimum_num_comments + ): + logger.info(f"{submission.title} <-- Number of comments too low!") + return False + + if "update" in submission.title.lower(): + return False + + if "covid" in submission.title.lower() or "vaccine" in submission.title.lower(): + logger.info( + f"{submission.title} <-- Youtube Channel Strikes if Covid content...!" + ) + return False + + return True + + +def posts(client_id: str, client_secret: str) -> List[Submission]: """Get a list of available Reddit posts. + Args: + client_id: Client ID used to access the Reddit API. + client_secret: Client secret used to access the Reddit API. + Returns: A list of zero or more Reddit submissions. """ - submissions: List[Submission] = get_reddit_submissions() + submissions: List[Submission] = get_reddit_submissions( + client_id=client_id, client_secret=client_secret + ) return submissions diff --git a/src/rybo/speech/__init__.py b/src/rybo/speech/__init__.py new file mode 100644 index 0000000..5b54a62 --- /dev/null +++ b/src/rybo/speech/__init__.py @@ -0,0 +1 @@ +"""Text to Speech processors.""" diff --git a/speech/speech.py b/src/rybo/speech/speech.py similarity index 86% rename from speech/speech.py rename to src/rybo/speech/speech.py index acebf6f..3e5cafe 100644 --- a/speech/speech.py +++ b/src/rybo/speech/speech.py @@ -12,69 +12,13 @@ from gtts import gTTS from moviepy.editor import AudioFileClip, concatenate_audioclips -import config -import config.settings as settings -from speech.streamlabs_polly import StreamlabsPolly -from speech.tiktok import TikTok -from utils.common import sanitize_text +from rybo import config +from rybo.config import settings +from rybo.speech.streamlabs_polly import StreamlabsPolly +from rybo.speech.tiktok import TikTok +from rybo.utils.common import sanitize_text -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) - - -def process_speech_text(text: str) -> str: - """Sanitize raw text. - - This will process raw text prior to converting it to speech, replacing - common abbreviations with their full english version to ensure that the - generated speech is intelligable. - - Args: - text: Text to be sanitized. - - Returns: - Updated text that has had common abbreviations converted to their - full english equivalent. - """ - text = text.replace(" AFAIK ", " as far as I know ") - text = text.replace("AITA", " Am I The Asshole? ") - text = text.replace(" AMA ", " Ask me anything ") - text = text.replace(" ELI5 ", " Explain Like I'm Five ") - text = text.replace(" IAMA ", " I am a ") - text = text.replace("IANAD", " i am not a doctor ") - text = text.replace("IANAL", " i am not a lawyer ") - text = text.replace(" IMO ", " in my opinion ") - text = text.replace(" NSFL ", " Not safe for life ") - text = text.replace(" NSFW ", " Not safe for Work ") - text = text.replace("NTA", " Not The Asshole ") - text = text.replace(" SMH ", " Shaking my head ") - text = text.replace("TD;LR", " too long didn't read ") - text = text.replace("TDLR", " too long didn't read ") - text = text.replace(" TIL ", " Today I Learned ") - text = text.replace("YTA", " You're the asshole ") - text = text.replace("SAHM", " stay at home mother ") - text = text.replace("WIBTA", " would I be the asshole ") - text = text.replace(" stfu ", " shut the fuck up ") - text = text.replace(" OP ", " o p ") - text = text.replace(" CB ", " choosing beggar ") - text = text.replace("pettyrevenge", "petty revenge") - text = text.replace("askreddit", "ask reddit") - text = text.replace("twoxchromosomes", "two x chromosomes") - text = text.replace("showerthoughts", "shower thoughts") - text = text.replace("amitheasshole", "am i the asshole") - text = text.replace("“", '"') - text = text.replace("“", '"') - text = text.replace("’", "'") - text = text.replace("...", ".") - text = text.replace("*", "") - text = re.sub(r"(\[|\()[0-9]{1,2}\s*(m|f)?(\)|\])", "", text, flags=re.IGNORECASE) - - text = sanitize_text(text) - return text +logger = logging.getLogger(__name__) def create_audio(path: Path, text: str) -> Path: @@ -87,7 +31,7 @@ def create_audio(path: Path, text: str) -> Path: Returns: Path to the generated audio file. """ - # logging.info(f"Generating Audio File : {text}") + # logger.info(f"Generating Audio File : {text}") output_path = os.path.normpath(path) text: str = process_speech_text(text) if not os.path.exists(output_path) or not os.path.getsize(output_path) > 0: @@ -114,7 +58,7 @@ def create_audio(path: Path, text: str) -> Path: speech_text_character_limit: int = 550 if len(text) > speech_text_character_limit: - logging.info( + logger.info( "Text exceeds StreamlabsPolly limit, breaking up into chunks" ) speech_chunks: List[Path] = [] @@ -124,16 +68,16 @@ def create_audio(path: Path, text: str) -> Path: break_long_words=True, break_on_hyphens=False, ) - print(chunk_list) + logger.info(chunk_list) for count, chunk in enumerate(chunk_list): - print(count) + logger.info(count) if chunk == "​": - logging.info("Skip zero space character comment : %s", chunk) + logger.info("Skip zero space character comment : %s", chunk) continue if chunk == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue tmp_path: Path = f"{output_path}{count}" @@ -144,7 +88,7 @@ def create_audio(path: Path, text: str) -> Path: final_clip: AudioFileClip = concatenate_audioclips(clips) final_clip.write_audiofile(output_path) else: - print(text) + logger.info(text) slp.run(text, output_path) if settings.voice_engine == "edge-tts": @@ -169,7 +113,7 @@ def create_audio(path: Path, text: str) -> Path: tt: TikTok = TikTok() if len(text) > speech_text_character_limit: - logging.info( + logger.info( "Text exceeds tiktok limit, \ breaking up into chunks" ) @@ -180,16 +124,16 @@ def create_audio(path: Path, text: str) -> Path: break_long_words=True, break_on_hyphens=False, ) - print(chunk_list) + logger.info(chunk_list) for count, chunk in enumerate(chunk_list): - print(count) + logger.info(count) if chunk == "​": - logging.info("Skip zero space character comment : %s", chunk) + logger.info("Skip zero space character comment : %s", chunk) continue if chunk == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue tmp_path: Path = f"{path}{count}" @@ -200,16 +144,67 @@ def create_audio(path: Path, text: str) -> Path: final_clip: AudioFileClip = concatenate_audioclips(clips) final_clip.write_audiofile(output_path) else: - print(text) + logger.info(text) tt.run(text, output_path) else: - logging.info("Audio file already exists : %s", output_path) + logger.info("Audio file already exists : %s", output_path) - logging.info("Created Audio File : %s", output_path) + logger.info("Created Audio File : %s", output_path) return output_path +def process_speech_text(text: str) -> str: + """Sanitize raw text. + + This will process raw text prior to converting it to speech, replacing + common abbreviations with their full english version to ensure that the + generated speech is intelligable. + + Args: + text: Text to be sanitized. + + Returns: + Updated text that has had common abbreviations converted to their + full english equivalent. + """ + text = text.replace(" AFAIK ", " as far as I know ") + text = text.replace("AITA", " Am I The Asshole? ") + text = text.replace(" AMA ", " Ask me anything ") + text = text.replace(" ELI5 ", " Explain Like I'm Five ") + text = text.replace(" IAMA ", " I am a ") + text = text.replace("IANAD", " i am not a doctor ") + text = text.replace("IANAL", " i am not a lawyer ") + text = text.replace(" IMO ", " in my opinion ") + text = text.replace(" NSFL ", " Not safe for life ") + text = text.replace(" NSFW ", " Not safe for Work ") + text = text.replace("NTA", " Not The Asshole ") + text = text.replace(" SMH ", " Shaking my head ") + text = text.replace("TD;LR", " too long didn't read ") + text = text.replace("TDLR", " too long didn't read ") + text = text.replace(" TIL ", " Today I Learned ") + text = text.replace("YTA", " You're the asshole ") + text = text.replace("SAHM", " stay at home mother ") + text = text.replace("WIBTA", " would I be the asshole ") + text = text.replace(" stfu ", " shut the fuck up ") + text = text.replace(" OP ", " o p ") + text = text.replace(" CB ", " choosing beggar ") + text = text.replace("pettyrevenge", "petty revenge") + text = text.replace("askreddit", "ask reddit") + text = text.replace("twoxchromosomes", "two x chromosomes") + text = text.replace("showerthoughts", "shower thoughts") + text = text.replace("amitheasshole", "am i the asshole") + text = text.replace("“", '"') + text = text.replace("“", '"') + text = text.replace("’", "'") + text = text.replace("...", ".") + text = text.replace("*", "") + text = re.sub(r"(\[|\()[0-9]{1,2}\s*(m|f)?(\)|\])", "", text, flags=re.IGNORECASE) + + text = sanitize_text(text) + return text + + if __name__ == "__main__": parser: ArgumentParser = ArgumentParser() parser.add_argument( @@ -220,6 +215,6 @@ def create_audio(path: Path, text: str) -> Path: ) args: Namespace = parser.parse_args() - print(args.path) - print(args.speech) + logger.info(args.path) + logger.info(args.speech) create_audio(args.path, args.speech) diff --git a/speech/streamlabs_polly.py b/src/rybo/speech/streamlabs_polly.py similarity index 94% rename from speech/streamlabs_polly.py rename to src/rybo/speech/streamlabs_polly.py index ae7a261..7fe71da 100644 --- a/speech/streamlabs_polly.py +++ b/src/rybo/speech/streamlabs_polly.py @@ -1,4 +1,5 @@ """Polly text to speech convertor.""" +import logging import sys import time as pytime from datetime import datetime @@ -8,14 +9,16 @@ from typing import Dict, List, Union import requests +from requests import Response +from requests.exceptions import JSONDecodeError + +from rybo.config import settings +from rybo.utils.common import sanitize_text # from utils import settings # from utils.voice import check_ratelimit -from requests import Response -from requests.exceptions import JSONDecodeError -import config.settings as settings -from utils.common import sanitize_text +logger = logging.getLogger(__name__) if sys.version_info[0] >= 3: from datetime import timezone @@ -40,67 +43,6 @@ # valid voices https://lazypy.ro/tts/ -def sleep_until(time: Union[int, datetime]) -> None: - """Pause until a specific end time. - - Args: - time: Either a valid datetime object or unix timestamp in seconds - (i.e. seconds since Unix epoch). - """ - end: int = time - - # Convert datetime to unix timestamp and adjust for locality - if isinstance(time, datetime): - # If we're on Python 3 and the user specified a timezone, convert to - # UTC and get the timestamp. - if sys.version_info[0] >= 3 and time.tzinfo: - end: datetime = time.astimezone(timezone.utc).timestamp() - else: - zone_diff: float = ( - pytime.time() - (datetime.now() - datetime(1970, 1, 1)).total_seconds() - ) - end: float = (time - datetime(1970, 1, 1)).total_seconds() + zone_diff - - # Type check - if not isinstance(end, (int, float)): - raise Exception("The time parameter is not a number or datetime object") - - # Now we wait - while True: - now: float = pytime.time() - diff: float = end - now - - # Time is up! - if diff <= 0: - break - else: - # 'logarithmic' sleeping to minimize loop iterations - sleep(diff / 2) - - -def check_ratelimit(response: Response) -> bool: - """Check if the rate limit has been hit. - - If it has, sleep for the time specified in the response. - - Args: - response: The HTTP response to be examined. - - Returns: - `True` if the rate limit has been reached, otherwise `False`. - """ - if response.status_code == 429: - try: - time: int = int(response.headers["X-RateLimit-Reset"]) - print("Ratelimit hit, sleeping...") - sleep_until(time) - return False - except KeyError: # if the header is not present, we don't know how long to wait - return False - - return True - - class StreamlabsPolly: """Polly text to speech convertor.""" @@ -110,6 +52,15 @@ def __init__(self): self.max_chars: int = 550 self.voices: List[str] = voices + def randomvoice(self) -> str: + """Select a random Polly voice. + + Returns: + The name of a randomly selected Polly voice. + """ + rnd: SystemRandom() = SystemRandom() + return rnd.choice(self.voices) + def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: """Convert text to an audio clip using a random Polly voice. @@ -145,13 +96,65 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: if response.json()["error"] == "No text specified!": raise ValueError("Please specify a text to convert to speech.") except (KeyError, JSONDecodeError): - print("Error occurred calling Streamlabs Polly") + logger.info("Error occurred calling Streamlabs Polly") - def randomvoice(self) -> str: - """Select a random Polly voice. - Returns: - The name of a randomly selected Polly voice. - """ - rnd: SystemRandom() = SystemRandom() - return rnd.choice(self.voices) +def check_ratelimit(response: Response) -> bool: + """Check if the rate limit has been hit. + + If it has, sleep for the time specified in the response. + + Args: + response: The HTTP response to be examined. + + Returns: + `True` if the rate limit has been reached, otherwise `False`. + """ + if response.status_code == 429: + try: + time: int = int(response.headers["X-RateLimit-Reset"]) + logger.info("Ratelimit hit, sleeping...") + sleep_until(time) + return False + except KeyError: # if the header is not present, we don't know how long to wait + return False + + return True + + +def sleep_until(time: Union[int, datetime]) -> None: + """Pause until a specific end time. + + Args: + time: Either a valid datetime object or unix timestamp in seconds + (i.e. seconds since Unix epoch). + """ + end: int = time + + # Convert datetime to unix timestamp and adjust for locality + if isinstance(time, datetime): + # If we're on Python 3 and the user specified a timezone, convert to + # UTC and get the timestamp. + if sys.version_info[0] >= 3 and time.tzinfo: + end: datetime = time.astimezone(timezone.utc).timestamp() + else: + zone_diff: float = ( + pytime.time() - (datetime.now() - datetime(1970, 1, 1)).total_seconds() + ) + end: float = (time - datetime(1970, 1, 1)).total_seconds() + zone_diff + + # Type check + if not isinstance(end, (int, float)): + raise Exception("The time parameter is not a number or datetime object") + + # Now we wait + while True: + now: float = pytime.time() + diff: float = end - now + + # Time is up! + if diff <= 0: + break + else: + # 'logarithmic' sleeping to minimize loop iterations + sleep(diff / 2) diff --git a/speech/tiktok.py b/src/rybo/speech/tiktok.py similarity index 95% rename from speech/tiktok.py rename to src/rybo/speech/tiktok.py index 162091c..aff2a1a 100644 --- a/speech/tiktok.py +++ b/src/rybo/speech/tiktok.py @@ -1,8 +1,9 @@ """TikTok text to speech convertor.""" import base64 -from random import SystemRandom +import logging import urllib.parse from pathlib import Path +from random import SystemRandom from typing import Dict, List import requests @@ -10,7 +11,9 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import SSLError -import config.settings as settings +from rybo.config import settings + +logger = logging.getLogger(__name__) # from profanity_filter import ProfanityFilter # pf = ProfanityFilter() @@ -87,6 +90,15 @@ def __init__(self): "noneng": noneng, } + def randomvoice(self) -> str: + """Select a random voice. + + Returns: + A randomly chosen human voice. + """ + rnd: SystemRandom = SystemRandom() + return rnd.choice(self.voices["human"]) + def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: """Convert text to an audio clip using a random tik-tok styled voice. @@ -107,9 +119,9 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: ) try: text: str = urllib.parse.quote(text) - print(len(text)) + logger.info(len(text)) tt_uri: str = f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" - print(tt_uri) + logger.info(tt_uri) r: Response = requests.post(tt_uri, timeout=120) except SSLError: # https://stackoverflow.com/a/47475019/18516611 @@ -121,25 +133,16 @@ def run(self, text: str, filepath: Path, random_voice: bool = False) -> None: r: Response = session.post( f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" ) - # print(r.text) + # logger.info(r.text) vstr: str = [r.json()["data"]["v_str"]][0] b64d: bytes = base64.b64decode(vstr) with open(filepath, "wb") as out: # noqa: SCS109 out.write(b64d) - def randomvoice(self) -> str: - """Select a random voice. - - Returns: - A randomly chosen human voice. - """ - rnd: SystemRandom = SystemRandom() - return rnd.choice(self.voices["human"]) - if __name__ == "__main__": tt: TikTok = TikTok() text_to_say: str = "Hello world this is some spoken text" - print(str(len(text_to_say))) + logger.info(str(len(text_to_say))) tt.run(text_to_say, "tiktok_test.mp3") diff --git a/src/rybo/thumbnail/__init__.py b/src/rybo/thumbnail/__init__.py new file mode 100644 index 0000000..b18e605 --- /dev/null +++ b/src/rybo/thumbnail/__init__.py @@ -0,0 +1 @@ +"""Thumbnail generators.""" diff --git a/thumbnail/keywords.py b/src/rybo/thumbnail/keywords.py similarity index 79% rename from thumbnail/keywords.py rename to src/rybo/thumbnail/keywords.py index a53906d..c94605b 100644 --- a/thumbnail/keywords.py +++ b/src/rybo/thumbnail/keywords.py @@ -1,8 +1,13 @@ """Functions to extract key phrases out of text.""" +import logging +from typing import List + from multi_rake import Rake +logger = logging.getLogger(__name__) + -def get_keywords(text): +def get_keywords(text) -> List[str]: """Extract key phrases out of a block of text. Args: @@ -11,8 +16,8 @@ def get_keywords(text): Returns: A selection of key phrases. """ - rake = Rake() - keywords = rake.apply(text) + rake: Rake = Rake() + keywords: List[str] = rake.apply(text) return keywords @@ -29,5 +34,5 @@ def get_keywords(text): "systems and systems of mixed types." ) - keywords = get_keywords(text) - print(keywords[:10]) + keywords: List[str] = get_keywords(text) + logger.info(keywords[:10]) diff --git a/thumbnail/lexica.py b/src/rybo/thumbnail/lexica.py similarity index 85% rename from thumbnail/lexica.py rename to src/rybo/thumbnail/lexica.py index 30eeebd..5357e74 100644 --- a/thumbnail/lexica.py +++ b/src/rybo/thumbnail/lexica.py @@ -7,10 +7,13 @@ from pathlib import Path from typing import List from urllib.request import Request, urlopen -from requests import Response + import requests +from requests import Response + +from rybo.config import settings -import config.settings as settings +logger = logging.getLogger(__name__) def download_image(url: str, file_path: Path) -> None: @@ -51,12 +54,12 @@ def get_images( if number_of_images > 0: safe_query: str = urllib.parse.quote(sentence.strip()) lexica_url: str = f"https://lexica.art/api/v1/search?q={safe_query}" - logging.info("Downloading Image From Lexica : %s", sentence) + logger.info("Downloading Image From Lexica : %s", sentence) try: r: Response = requests.get(lexica_url, timeout=120) j: object = json.loads(r.text) except Exception: - print("Error Retrieving Lexica Images") + logger.info("Error Retrieving Lexica Images") pass return @@ -66,9 +69,9 @@ def get_images( image_url: str = j["images"][num]["src"] download_image(image_url, image_path) else: - logging.info("Image already exists : %s - %s", id, sentence) + logger.info("Image already exists : %s - %s", id, sentence) images.append(image_path) else: - logging.info("Downloading Images Set to False.......") + logger.info("Downloading Images Set to False.......") return images diff --git a/thumbnail/thumbnail.py b/src/rybo/thumbnail/thumbnail.py similarity index 92% rename from thumbnail/thumbnail.py rename to src/rybo/thumbnail/thumbnail.py index eec871c..922602a 100644 --- a/thumbnail/thumbnail.py +++ b/src/rybo/thumbnail/thumbnail.py @@ -7,24 +7,20 @@ import os import sys from pathlib import Path - -# from nltk.corpus import stopwords from random import SystemRandom from typing import Any, List from moviepy.editor import CompositeVideoClip, ImageClip, TextClip from PIL import Image -import config.settings as settings -import thumbnail.lexica as lexica -from utils.common import random_rgb_colour, sanitize_text +from rybo.config import settings +from rybo.thumbnail import lexica +from rybo.utils.common import random_rgb_colour, sanitize_text + +# from nltk.corpus import stopwords + -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], -) +logger = logging.getLogger(__name__) def apply_black_gradient( @@ -64,7 +60,7 @@ def apply_black_gradient( alpha_gradient.putpixel((x, 0), a) else: alpha_gradient.putpixel((x, 0), 0) - # print '{}, {:.2f}, {}'.format(x, float(x) / width, a) + # logger.info '{}, {:.2f}, {}'.format(x, float(x) / width, a) alpha: Any = alpha_gradient.resize(input_im.size) # create black image, apply gradient @@ -87,8 +83,8 @@ def get_font_size(length: int) -> int: Returns: A suggested font size. """ - fontsize = 50 - lineheight = 60 + fontsize: int = 50 + lineheight: int = 60 if length < 10: fontsize = 190 @@ -120,9 +116,9 @@ def get_font_size(length: int) -> int: if length >= 90 and length < 100: fontsize = 80 - logging.debug("Title Length : %s", length) - logging.debug("Setting Fontsize : %s", fontsize) - logging.debug("Setting Lineheight : %s", lineheight) + logger.debug("Title Length : %s", length) + logger.debug("Setting Fontsize : %s", fontsize) + logger.debug("Setting Lineheight : %s", lineheight) return fontsize, lineheight @@ -148,7 +144,7 @@ def generate( Returns: A list of paths where the downloaded images can be found. """ - logging.info("========== Generating Thumbnail ==========") + logger.info("========== Generating Thumbnail ==========") # image_path = str(Path(video_directory, "lexica.png").absolute()) @@ -158,7 +154,7 @@ def generate( title = title.replace(" ", " ") title = title.replace("’", "") - logging.info(title) + logger.info(title) if settings.thumbnail_image_source == "random": rnd: SystemRandom = SystemRandom() @@ -173,7 +169,7 @@ def generate( video_directory, title, number_of_images=number_of_thumbnails ) - thumbnails = [] + thumbnails: List[Path] = [] if images: for index, image in enumerate(images): @@ -206,7 +202,7 @@ def create_thumbnail( video_directory, f"thumbnail_{str(index)}.png" ).absolute() if thumbnail_path.exists(): - logging.info("Thumbnail already exists : %s", thumbnail_path) + logger.info("Thumbnail already exists : %s", thumbnail_path) return thumbnail_path border_width: int = 20 @@ -271,8 +267,8 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: # fontsize = 40 title = title.replace("’", "") fontsize, lineheight = get_font_size(len(title)) - logging.info("Title Length : %s", len(title)) - logging.info("Optimising Font Size : ") + logger.info("Title Length : %s", len(title)) + logger.info("Optimising Font Size : ") sys.stdout.write(str(fontsize)) while True: @@ -282,7 +278,7 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: sys.stdout.write(".") if txt_clip.h > height: optimal_font_size: int = previous_fontsize - print(optimal_font_size) + logger.info(optimal_font_size) break word_height: TextClip = get_text_clip( @@ -377,7 +373,7 @@ def get_text_clip(text: str, fs: int, txt_color: str = "#FFFFFF") -> TextClip: final_video: CompositeVideoClip = CompositeVideoClip(clips) final_video = final_video.margin(border_width, color=random_rgb_colour()) - logging.info("Saving Thumbnail to : %s", thumbnail_path) + logger.info("Saving Thumbnail to : %s", thumbnail_path) final_video.save_frame(thumbnail_path, 1) return thumbnail_path diff --git a/src/rybo/utils/__init__.py b/src/rybo/utils/__init__.py new file mode 100644 index 0000000..5c7ffd7 --- /dev/null +++ b/src/rybo/utils/__init__.py @@ -0,0 +1,109 @@ +"""Helper utilities.""" + +import ast +import os +from argparse import Action, ArgumentParser, Namespace +from typing import Any, Optional, Sequence, Union + + +class EnvDefault(Action): + """Custom action used to set CLI arguments via environment variables.""" + + def __init__(self, envvar: str, required=False, default=None, **kwargs) -> None: + """Initialize the action. + + Args: + envvar: Name of the environment variable to read the argument value from. + required: ```True``` to make the argument mandatory, otherwise ```False```. + default: Default value is none is specified by the user. + kwargs: Extra argument configuration options. + """ + if not default and envvar: + default = os.getenv(envvar, None) + + if required and default: + required = False + + super(EnvDefault, self).__init__(default=default, required=required, **kwargs) + + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Union[str, Sequence[Any]]], + option_string: Optional[str] = None, + ) -> None: + """Class setup executed during object instantiation. + + Args: + parser: ArgumentParser that contains the user specified CLI parameters. + namespace: Namespace that contains the argument parser parameters. + values: Parameter value. + option_string: Option strings passed to the parameter. + """ + setattr(namespace, self.dest, values) + + +class EnvFlagDefault(Action): + """Custom action used to set CLI flags via environment variables.""" + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: Optional[bool] = None, + required: bool = False, + help: Optional[str] = None, + envvar: Optional[str] = None, + metavar: Optional[str] = None, + ) -> None: + """Initialize the action. + + If a default value has not been defined, but an environment variable name has + been provided, the action will try to set the flag based on the truthiness of + the provided environment variable (True/False). + + Args: + option_strings: Raw parameter options. + dest: Name of the destination parameter that the parameter value will be + stored under. + default: Default value is none is specified by the user. + required: ```True``` to make the argument mandatory, otherwise ```False```. + help: Help text displayed when the -h/--help parameter is used. + envvar: Name of the environment variable to read the argument value from. + metavar: Name of the metavar used as a placeholder on the help screen. + """ + if not default and envvar: + default = ast.literal_eval(os.getenv(envvar, "False")) + + if required and default: + required = False + + super(EnvFlagDefault, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help, + metavar=metavar, + nargs=0, + ) + self.envvar = envvar + + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Union[str, Sequence[Any]]], + option_string: Optional[str] = None, + ) -> None: + """Class setup executed during object instantiation. + + Args: + parser: ArgumentParser that contains the user specified CLI parameters. + namespace: Namespace that contains the argument parser parameters. + values: Parameter value. + option_string: Option strings passed to the parameter. + """ + setattr(namespace, self.dest, self.const) diff --git a/utils/base64_encoding.py b/src/rybo/utils/base64_encoding.py similarity index 86% rename from utils/base64_encoding.py rename to src/rybo/utils/base64_encoding.py index 0e6848e..99d3a7f 100644 --- a/utils/base64_encoding.py +++ b/src/rybo/utils/base64_encoding.py @@ -1,5 +1,8 @@ """Base64 encoding utilities.""" import base64 +import logging + +logger = logging.getLogger(__name__) def base64_encode_string(message: str) -> None: @@ -12,7 +15,7 @@ def base64_encode_string(message: str) -> None: message_bytes = message.encode("ascii") base64_bytes = base64.b64encode(message_bytes) base64_message = base64_bytes.decode("ascii") - print(base64_message) + logger.info(base64_message) def base64_decode_string(message: str) -> str: @@ -34,5 +37,5 @@ def base64_decode_string(message: str) -> str: # Encoded keywords. Testing the decoding function? message = "cG9ybixzZXgsamVya2luZyBvZmYsc2x1dCxyYXBl" decoded_string = base64_decode_string(message) -print(decoded_string) -print(decoded_string.split(",")) +logger.info(decoded_string) +logger.info(decoded_string.split(",")) diff --git a/utils/common.py b/src/rybo/utils/common.py similarity index 96% rename from utils/common.py rename to src/rybo/utils/common.py index 9384e18..d73c152 100644 --- a/utils/common.py +++ b/src/rybo/utils/common.py @@ -1,8 +1,11 @@ """Common helpers to keep the codebase DRY.""" -from random import SystemRandom -import re +import logging import os +import re from pathlib import Path +from random import SystemRandom + +logger = logging.getLogger(__name__) def random_hex_colour() -> str: @@ -25,8 +28,8 @@ def random_rgb_colour() -> int: """ rnd: SystemRandom = SystemRandom() rbg_colour: int = rnd.choices(range(256), k=3) - print("Generated random rgb colour") - print(rbg_colour) + logger.info("Generated random rgb colour") + logger.info(rbg_colour) return rbg_colour diff --git a/csvmgr.py b/src/rybo/utils/csvmgr.py similarity index 88% rename from csvmgr.py rename to src/rybo/utils/csvmgr.py index 2431c64..12ff96c 100644 --- a/csvmgr.py +++ b/src/rybo/utils/csvmgr.py @@ -1,10 +1,12 @@ """Manage the CSV file that contains metadata about Reddit posts.""" +import logging import os -from typing import Any, Dict +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame -from typing import List + +logger = logging.getLogger(__name__) class CsvWriter: @@ -50,7 +52,9 @@ def is_uploaded(self, id: str) -> bool: csv: DataFrame = pd.read_csv(self.csv_file) # TODO: ??? - results: int = len(csv.loc[(csv["id"] == id) & (csv["uploaded"] == True)]) + results: int = len( + csv.loc[(csv["id"] == id) & (csv["uploaded"] == True)] # noqa: E712 + ) if results > 0: return True else: @@ -72,7 +76,7 @@ def set_uploaded(self, id: str) -> None: if __name__ == "__main__": csvwriter: CsvWriter = CsvWriter() csvwriter.initialise_csv() - print(csvwriter.is_uploaded("snppah")) + logger.info(csvwriter.is_uploaded("snppah")) csvwriter.set_uploaded("snppah") - print(csvwriter.is_uploaded("snppah")) + logger.info(csvwriter.is_uploaded("snppah")) diff --git a/utils/speed_test.py b/src/rybo/utils/speed_test.py similarity index 100% rename from utils/speed_test.py rename to src/rybo/utils/speed_test.py diff --git a/src/rybo/video_generation/__init_-.py b/src/rybo/video_generation/__init_-.py new file mode 100644 index 0000000..e70d862 --- /dev/null +++ b/src/rybo/video_generation/__init_-.py @@ -0,0 +1 @@ +"""Video processors.""" diff --git a/video_generation/video.py b/src/rybo/video_generation/video.py similarity index 75% rename from video_generation/video.py rename to src/rybo/video_generation/video.py index 883d2e2..da54a38 100644 --- a/video_generation/video.py +++ b/src/rybo/video_generation/video.py @@ -7,21 +7,15 @@ import json import logging import os +import sys from os import path from pathlib import Path from random import SystemRandom from typing import Any, Dict, List, Optional -from comments.screenshot import ( - download_screenshot_of_reddit_post_title, - download_screenshots_of_reddit_posts, -) - -import config.settings as settings - -import csvmgr - +import colorama import moviepy.video.fx.all as vfx +from colorama import Fore from moviepy.editor import ( AudioFileClip, ColorClip, @@ -30,24 +24,22 @@ TextClip, VideoFileClip, ) - from praw.models import Comment, Submission from praw.models.comment_forest import CommentForest -import publish.youtube as youtube - -import speech.speech as speech - -from thumbnail.thumbnail import get_font_size - -from utils.common import contains_url, give_emoji_free_text, sanitize_text - -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[logging.FileHandler("debug.log", "w", "utf-8"), logging.StreamHandler()], +from rybo.comments.screenshot import ( + download_screenshot_of_reddit_post_title, + download_screenshots_of_reddit_posts, ) +from rybo.config import settings +from rybo.publish import youtube +from rybo.speech import speech +from rybo.thumbnail.thumbnail import get_font_size +from rybo.utils import csvmgr +from rybo.utils.common import contains_url, give_emoji_free_text, sanitize_text + +logger = logging.getLogger(__name__) +colorama.init(autoreset=True) def log_group_header(title: str) -> str: @@ -81,13 +73,13 @@ def print_post_details(post: Submission) -> None: Args: post: The Reddit post. """ - logging.info("SubReddit : %s", post.subreddit_name_prefixed) - logging.info("Title : %s", post.title) - logging.info("Score : %s", post.score) - logging.info("ID : %s", post.id) - logging.info("URL : %s", post.url) - logging.info("SelfText : %s", post.selftext) - logging.info("NSFW? : %s", post.over_18) + logger.info("SubReddit : %s", post.subreddit_name_prefixed) + logger.info("Title : %s", post.title) + logger.info("Score : %s", post.score) + logger.info("ID : %s", post.id) + logger.info("URL : %s", post.url) + logger.info("SelfText : %s", post.selftext) + logger.info("NSFW? : %s", post.over_18) def print_comment_details(comment: Comment) -> None: @@ -97,11 +89,11 @@ def print_comment_details(comment: Comment) -> None: comment: The comment. """ if comment.author: - logging.debug("Author : %s", comment.author) - logging.debug("id : %s", comment.id) - logging.debug("Stickied : %s", comment.stickied) - logging.info("Comment : %s", give_emoji_free_text(str(comment.body))) - logging.info("Length : %s", len(comment.body)) + logger.debug("Author : %s", comment.author) + logger.debug("id : %s", comment.id) + logger.debug("Stickied : %s", comment.stickied) + logger.info("Comment : %s", give_emoji_free_text(str(comment.body))) + logger.info("Length : %s", len(comment.body)) class Video: @@ -160,7 +152,7 @@ def get_background(self) -> None: """Select a random background for the video.""" rnd: SystemRandom = SystemRandom() self.background = rnd.choice(seq=os.listdir(settings.background_directory)) - logging.info("Randomly Selecting Background : %s", self.background) + logger.info("Randomly Selecting Background : %s", self.background) def compile(self) -> None: """Compile the video. @@ -189,7 +181,13 @@ def get_random_lines(file_name: Path, num_lines: int) -> str: return "\n".join(random_lines) -def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> None: +def create( + video_directory: Path, + post: Submission, + thumbnails: List[Path], + username: str, + password: str, +) -> None: """Generate a video from a processed reddit post. Args: @@ -197,8 +195,10 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N post: Reddit post that's the main topic of the video. thumbnails: List of images to be embedded in the video. For example, screenshots of user comments. + username: Reddit username. + password: Reddit password. """ - logging.info(log_group_header(title="Processing Reddit Post")) + logger.info(log_group_header(title="Processing Reddit Post")) print_post_details(post) v = Video() @@ -282,12 +282,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N .set_opacity(settings.reddit_comment_opacity) ) if title_clip.w > title_clip.h: - print("Resizing Horizontally") + logger.info("Resizing Horizontally") title_clip = title_clip.resize( width=settings.video_width * settings.reddit_comment_width ) else: - print("Resizing Vertically") + logger.info("Resizing Vertically") title_clip = title_clip.resize(height=settings.video_height * 0.95) else: title_clip = ( @@ -308,14 +308,14 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N newcaster_start: int = t if v.meta.selftext and settings.enable_selftext: - logging.info(log_group_header(title="Processing SelfText")) - logging.info(v.meta.selftext) + logger.info(log_group_header(title="Processing SelfText")) + logger.info(v.meta.selftext) selftext: str = sanitize_text(v.meta.selftext) selftext = give_emoji_free_text(selftext) selftext = os.linesep.join([s for s in selftext.splitlines() if s]) - logging.debug("selftext Length : %s", len(selftext)) + logger.debug("selftext Length : %s", len(selftext)) selftext_lines: List[str] = selftext.splitlines() @@ -327,8 +327,8 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N if selftext_line == " " or selftext_line == " ": continue - logging.debug("selftext length : %s", len(selftext_line)) - logging.debug("selftext_line : %s", selftext_line) + logger.debug("selftext length : %s", len(selftext_line)) + logger.debug("selftext_line : %s", selftext_line) selftext_audio_filepath: str = str( Path(speech_directory, f"selftext_{str(selftext_line_count)}.mp3") ) @@ -336,32 +336,38 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N selftext_audioclip: AudioFileClip = AudioFileClip(selftext_audio_filepath) current_clip_text += f"{selftext_line}\n" - logging.debug("Current Clip Text :") - logging.debug(current_clip_text) - logging.debug("SelfText Fontsize : %s", settings.text_fontsize) - - selftext_clip: TextClip = ( - TextClip( - current_clip_text, - font=settings.text_font, - fontsize=settings.text_fontsize, - color=settings.text_color, - size=txt_clip_size, - kerning=-1, - method="caption", - # bg_color=settings.text_bg_color, - align="West", + logger.debug("Current Clip Text :") + logger.debug(current_clip_text) + logger.debug("SelfText Fontsize : %s", settings.text_fontsize) + + try: + selftext_clip: TextClip = ( + TextClip( + current_clip_text, + font=settings.text_font, + fontsize=settings.text_fontsize, + color=settings.text_color, + size=txt_clip_size, + kerning=-1, + method="caption", + # bg_color=settings.text_bg_color, + align="West", + ) + .set_position((clip_margin, clip_margin_top)) + .set_duration(selftext_audioclip.duration + settings.pause) + .set_audio(selftext_audioclip) + .set_start(t) + .set_opacity(settings.text_bg_opacity) + .volumex(1.5) ) - .set_position((clip_margin, clip_margin_top)) - .set_duration(selftext_audioclip.duration + settings.pause) - .set_audio(selftext_audioclip) - .set_start(t) - .set_opacity(settings.text_bg_opacity) - .volumex(1.5) - ) + except IOError as ioerr: + logger.exception( + f"{Fore.RED}An unexpected error has occured.", exc_info=ioerr + ) + sys.exit(1) if selftext_clip.h > settings.video_height: - logging.debug("Text exceeded Video Height, reset text") + logger.debug("Text exceeded Video Height, reset text") current_clip_text = f"{selftext_line}\n" selftext_clip = ( TextClip( @@ -383,18 +389,18 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if selftext_clip.h > settings.video_height: - logging.debug("Comment Text Too Long, Skipping Comment") + logger.debug("Comment Text Too Long, Skipping Comment") continue t += selftext_audioclip.duration + settings.pause v.duration += selftext_audioclip.duration + settings.pause v.clips.append(selftext_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) - logging.info(log_group_header(title="Finished Processing SelfText")) + logger.info("Current Video Duration : %s", v.duration) + logger.info(log_group_header(title="Finished Processing SelfText")) static_clip: VideoFileClip = ( VideoFileClip("static.mp4") @@ -419,16 +425,16 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N rejected_comments: List[Comment] = [] - logging.info(log_group_header(title="Filtering Reddit Comments")) + logger.info(log_group_header(title="Filtering Reddit Comments")) for count, c in enumerate(all_comments): - logging.info(log_group_subheader(title=f"Comment # {str(count)}")) + logger.info(log_group_subheader(title=f"Comment # {str(count)}")) print_comment_details(c) comment: str = c.body if len(comment) > settings.comment_length_max: - logging.info( + logger.info( "Status : REJECTED, Comment exceeds max character length : %s", settings.comment_length_max, ) @@ -436,12 +442,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N continue if comment == "[removed]" or comment == "[deleted]": - logging.info("Status : REJECTED, Skipping Comment : %s", comment) + logger.info("Status : REJECTED, Skipping Comment : %s", comment) rejected_comments.append(c) continue if "covid" in comment.lower() or "vaccine" in comment.lower(): - logging.info( + logger.info( "Status : REJECTED, Covid related, \ Youtube will Channel Strike..: %s", comment, @@ -452,24 +458,24 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N comment = give_emoji_free_text(comment) comment = os.linesep.join([s for s in comment.splitlines() if s]) - logging.debug("Comment Length : %s", len(comment)) + logger.debug("Comment Length : %s", len(comment)) if c.stickied: - logging.info("Status : REJECTED, Skipping Stickied Comment...") + logger.info("Status : REJECTED, Skipping Stickied Comment...") rejected_comments.append(c) continue if contains_url(comment): - logging.info("Status : REJECTED, Skipping Comment with URL in it...") + logger.info("Status : REJECTED, Skipping Comment with URL in it...") rejected_comments.append(c) continue - logging.info("Status : ACCEPTED") + logger.info("Status : ACCEPTED") accepted_comments.append(c) if len(accepted_comments) == settings.comment_limit: - logging.info("Rejected Comments : %s", len(rejected_comments)) - logging.info("Accepted Comments : %s", len(accepted_comments)) + logger.info("Rejected Comments : %s", len(rejected_comments)) + logger.info("Accepted Comments : %s", len(accepted_comments)) break screenshot_directory = Path(settings.screenshot_directory, v.meta.id) if settings.commentstyle == "reddit": @@ -477,10 +483,12 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N accepted_comments, f"http://reddit.com{v.meta.permalink}", screenshot_directory, + username, + password, ) for count, accepted_comment in enumerate(accepted_comments): - logging.info( + logger.info( "=== Processing Reddit Comment %s/%s ===", count, len(accepted_comments) ) @@ -509,22 +517,22 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) ) except Exception as e: - print(e) + logger.info(e) continue else: - logging.info("Comment image not found : %s", img_path) + logger.info("Comment image not found : %s", img_path) continue if img_clip.h > settings.video_height: - logging.info("Comment larger than video height : %s", img_path) + logger.info("Comment larger than video height : %s", img_path) continue if v.duration + audioclip.duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length ) - logging.info("Used %s/%s comments", count, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", count, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break t += audioclip.duration + settings.pause @@ -532,23 +540,23 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N v.clips.append(img_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) + logger.info("Current Video Duration : %s", v.duration) if settings.commentstyle == "text": comment_lines: List[str] = accepted_comment.body.splitlines() for ccount, comment_line in enumerate(comment_lines): if comment_line == "​": - logging.info("Skip zero space character comment : %s", comment) + logger.info("Skip zero space character comment : %s", comment) continue if comment_line == "": - logging.info("Skipping blank comment") + logger.info("Skipping blank comment") continue - logging.debug("comment_line : %s", comment_line) + logger.debug("comment_line : %s", comment_line) audio_filepath = str( Path( speech_directory, @@ -559,8 +567,8 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N audioclip = AudioFileClip(audio_filepath) current_clip_text += f"{comment_line}\n\n" - logging.debug("Current Clip Text :") - logging.debug(current_clip_text) + logger.debug("Current Clip Text :") + logger.debug(current_clip_text) txt_clip: TextClip = ( TextClip( @@ -583,7 +591,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if txt_clip.h > settings.video_height: - logging.debug("Text exceeded Video Height, reset text") + logger.debug("Text exceeded Video Height, reset text") current_clip_text = f"{comment_line}\n\n" txt_clip = ( TextClip( @@ -605,58 +613,58 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if txt_clip.h > settings.video_height: - logging.debug("Comment Text Too Long, Skipping Comment") + logger.debug("Comment Text Too Long, Skipping Comment") continue total_duration: int = v.duration + audioclip.duration if total_duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length, ) - logging.info( + logger.info( "Used %s/%s comments", ccount, len(accepted_comments), ) - logging.info("=== Finished Processing Comments ===") + logger.info("=== Finished Processing Comments ===") break t += audioclip.duration + settings.pause v.duration += audioclip.duration + settings.pause v.clips.append(txt_clip) - logging.debug("Video Clips : ") - logging.debug(str(len(v.clips))) + logger.debug("Video Clips : ") + logger.debug(str(len(v.clips))) - logging.info("Current Video Duration : %s", v.duration) + logger.info("Current Video Duration : %s", v.duration) if v.duration > settings.max_video_length: - logging.info( + logger.info( "Reached Maximum Video Length : %s", settings.max_video_length ) - logging.info("Used %s/%s comments", ccount, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", ccount, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break if count == settings.comment_limit: - logging.info( + logger.info( "Reached Maximum Number of Comments Limit : %s", settings.comment_limit, ) - logging.info("Used %s/%s comments", ccount, len(accepted_comments)) - logging.info("=== Finished Processing Comments ===") + logger.info("Used %s/%s comments", ccount, len(accepted_comments)) + logger.info("=== Finished Processing Comments ===") break else: - logging.info("Skipping comments!") + logger.info("Skipping comments!") - logging.info(log_group_subheader(title="Adding Background Clip")) + logger.info(log_group_subheader(title="Adding Background Clip")) if settings.enable_background: background_filepath: Path = Path( settings.background_directory, str(v.background) ) - logging.info("Background : %s", background_filepath) + logger.info("Background : %s", background_filepath) background_clip: VideoFileClip = ( VideoFileClip(background_filepath) @@ -666,24 +674,24 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if settings.orientation == "portrait": - print("Portrait mode, cropping and resizing!") + logger.info("Portrait mode, cropping and resizing!") background_clip = background_clip.crop( x1=1166.6, y1=0, x2=2246.6, y2=1920 ).resize((settings.vertical_video_width, settings.vertical_video_height)) if background_clip.duration < v.duration: - logging.debug("Looping Background") + logger.debug("Looping Background") # background_clip = vfx.make_loopable(background_clip, cross=0) background_clip = vfx.loop( background_clip, duration=v.duration ).without_audio() video_duration: str = str(background_clip.duration) - logging.debug("Looped Background Clip Duration : %s", video_duration) + logger.debug("Looped Background Clip Duration : %s", video_duration) else: - logging.debug("Not Looping Background") + logger.debug("Not Looping Background") background_clip = background_clip.set_duration(v.duration) else: - logging.info("Background not enabled...") + logger.info("Background not enabled...") background_clip = ColorClip( size=(settings.video_width, settings.video_height), color=settings.background_colour, @@ -692,7 +700,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N v.clips.insert(0, background_clip) if settings.enable_overlay: - logging.info(log_group_subheader(title="Adding Overlay Clip")) + logger.info(log_group_subheader(title="Adding Overlay Clip")) clip_video_overlay: VideoFileClip = ( VideoFileClip(settings.video_overlay_filepath) .set_start(tb) @@ -702,22 +710,22 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if clip_video_overlay.duration < v.duration: - logging.debug("Looping Overlay") + logger.debug("Looping Overlay") # background_clip = vfx.make_loopable(background_clip, cross=0) clip_video_overlay = vfx.loop( clip_video_overlay, duration=v.duration ).without_audio() video_duration = str(clip_video_overlay.duration) - logging.debug("Looped Overlay Clip Duration : %s", video_duration) + logger.debug("Looped Overlay Clip Duration : %s", video_duration) else: - logging.debug("Not Looping Overlay") + logger.debug("Not Looping Overlay") clip_video_overlay = clip_video_overlay.set_duration(v.duration) v.clips.insert(1, clip_video_overlay) if settings.enable_newscaster and settings.newscaster_filepath: - logging.info(log_group_subheader(title="Adding Newcaster Clip")) - logging.info("Newscaster File Path: %s", settings.newscaster_filepath) + logger.info(log_group_subheader(title="Adding Newcaster Clip")) + logger.info("Newscaster File Path: %s", settings.newscaster_filepath) clip_video_newscaster: VideoFileClip = ( VideoFileClip(settings.newscaster_filepath) .set_position(settings.newscaster_position) @@ -728,7 +736,7 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if settings.newscaster_remove_greenscreen: - logging.info(log_group_subheader(title="Removing Newcaster Green Screen")) + logger.info(log_group_subheader(title="Removing Newcaster Green Screen")) # Green Screen Video https://github.com/Zulko/moviepy/issues/964 clip_video_newscaster = clip_video_newscaster.fx( vfx.mask_color, @@ -738,15 +746,15 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N ) if clip_video_newscaster.duration < v.duration: - logging.debug("Looping Newscaster") + logger.debug("Looping Newscaster") clip_video_newscaster = vfx.loop( clip_video_newscaster, duration=v.duration - newcaster_start ).without_audio() - logging.debug( + logger.debug( "Looped Newscaster Clip Duration : %s", clip_video_newscaster.duration ) else: - logging.debug("Not Looping Newscaster") + logger.debug("Not Looping Newscaster") clip_video_newscaster = clip_video_newscaster.set_duration( v.duration - newcaster_start ) @@ -786,31 +794,31 @@ def create(video_directory: Path, post: Submission, thumbnails: List[Path]) -> N csvwriter.write_entry(row=row) if settings.enable_compilation: - logging.info(log_group_subheader(title="Compiling Video Clip")) - logging.info("Compiling video, this takes a while, please be patient : ") + logger.info(log_group_subheader(title="Compiling Video Clip")) + logger.info("Compiling video, this takes a while, please be patient : ") post_video.write_videofile(v.filepath, fps=24) else: - logging.info("Skipping Video Compilation --enable_compilation passed") + logger.info("Skipping Video Compilation --enable_compilation passed") if settings.enable_compilation and settings.enable_upload: if path.exists("client_secret.json") and path.exists("credentials.storage"): if csvwriter.is_uploaded(v.meta.id): - logging.info("Already uploaded according to data.csv") + logger.info("Already uploaded according to data.csv") else: - logging.info( + logger.info( log_group_subheader(title="Uploading Video Clip to YouTube") ) try: youtube.publish(v) except Exception as e: - print(e) + logger.info(e) else: csvwriter.set_uploaded(v.meta.id) else: - logging.info( + logger.info( "Skipping upload, missing either \ client_secret.json or credentials.storage file." ) else: - logging.info("Skipping Upload...") + logger.info("Skipping Upload...")