diff --git a/cache_stats.py b/cache_stats.py index a835313..9e54642 100755 --- a/cache_stats.py +++ b/cache_stats.py @@ -11,6 +11,7 @@ import rss2irc from lib import CachedData +from lib import utils BUCKET_COUNT = 10 @@ -57,7 +58,9 @@ def calc_distribution( def get_timestamp(data) -> int: """Convert input data to int. - :raises: KeyError, TypeError, ValueError + :raises KeyError: raised when expected key is not found in data + :raises TypeError: raised for unsupported data types + :raises ValueError: raised when conversion of value to int fails """ if isinstance(data, (int, float)): return int(data) @@ -85,11 +88,8 @@ def get_timestamp_minmax( logger.debug("%s", traceback.format_exc()) continue - if timestamp < ts_min: - ts_min = timestamp - - if timestamp > ts_max: - ts_max = timestamp + ts_min = min(ts_min, timestamp) + ts_max = max(ts_max, timestamp) return ts_min, ts_max, error_cnt @@ -119,15 +119,16 @@ def generate_buckets( def main(): """Read cache file and print-out stats.""" + args = parse_args() logging.basicConfig(level=logging.INFO, stream=sys.stdout) logger = logging.getLogger("cache_stats") - args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) + logger.setLevel(args.log_level) - cache = rss2irc.read_cache(logger, args.cache) + cache = rss2irc.read_cache(logger, args.cache_file) logger.info( - "Number of items in cache '%s' is %d.", args.cache, len(cache.items) + "Number of items in cache '%s' is %d.", + args.cache_file, + len(cache.items), ) if not cache.items: logger.info("Nothing to do.") @@ -168,23 +169,25 @@ def main(): def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) parser.add_argument( "--cache", - dest="cache", + dest="cache_file", type=str, default=None, required=True, help="File which contains cache.", ) - return parser.parse_args() + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Increase log level verbosity. Can be passed multiple times.", + ) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + return args if __name__ == "__main__": diff --git a/ci/run-reorder-python-imports.sh b/ci/run-reorder-python-imports.sh index 17cf4e8..6bf2f57 100755 --- a/ci/run-reorder-python-imports.sh +++ b/ci/run-reorder-python-imports.sh @@ -5,4 +5,4 @@ set -u cd "$(dirname "${0}")/.." find . ! -path '*/\.*' -name '*.py' -print0 | \ - xargs -0 -- reorder-python-imports --py311-plus + xargs -0 -- reorder-python-imports --py313-plus diff --git a/gh2slack.py b/gh2slack.py index fb91f33..f3c0213 100755 --- a/gh2slack.py +++ b/gh2slack.py @@ -8,18 +8,21 @@ import re import sys import time -import traceback import urllib.parse +from dataclasses import dataclass from typing import Dict from typing import List from typing import Set import requests -import rss2irc # noqa: I202 -import rss2slack # noqa: I202 -from lib import CachedData # noqa: I202 -from lib import config_options # noqa: I202 +import rss2irc +import rss2slack +from lib import CachedData +from lib import cli_args +from lib import config_options +from lib import utils +from lib.exceptions import SlackTokenError ALIASES = { "issues": "issue", @@ -30,6 +33,15 @@ RE_LINK_REL_NEXT = re.compile(r"<(?P.*)>; rel=\"next") +@dataclass +class GHRepoInfo: + """Class holds information about GitHub repository.""" + + repo_owner: str + repo_name: str + repo_section: str + + def format_message( logger: logging.Logger, owner: str, @@ -42,11 +54,10 @@ def format_message( try: title = cache_item["title"].encode("utf-8") except UnicodeEncodeError: - logger.error( + logger.exception( "Failed to encode title as UTF-8: %s", repr(cache_item.get("title", None)), ) - logger.error("%s", traceback.format_exc()) title = "Unknown title due to UTF-8 exception, {:s}#{:d}".format( section, cache_item["number"] ) @@ -62,7 +73,7 @@ def format_message( title.decode("utf-8"), ) except UnicodeDecodeError: - logger.error("Failed to format message: %s", traceback.format_exc()) + logger.exception("Failed to format message.") message = "[{:s}/{:s}] Failed to format message for {:s}#{:d}".format( owner, repo, section, cache_item["number"] ) @@ -111,7 +122,7 @@ def gh_request( responses. """ logger.debug("Requesting %s", url) - user_agent = "gh2slack_{:d}".format(int(time.time())) + user_agent = "gh2slack-script" rsp = requests.get( url, headers={ @@ -147,17 +158,21 @@ def gh_request( def main(): """Fetch issues/PRs from GitHub and post them to Slack.""" + args = parse_args() logging.basicConfig(stream=sys.stdout, level=logging.ERROR) logger = logging.getLogger("gh2slack") - args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) + logger.setLevel(args.log_level) + + retcode = 0 + cache = rss2irc.wrap_read_cache(logger, args.cache_file) + if cache is None: + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) try: slack_token = rss2slack.get_slack_token() url = get_gh_api_url(args.gh_owner, args.gh_repo, args.gh_section) pages = gh_request(logger, url) - logger.debug("Got %i pages from GH.", len(pages)) if not pages: logger.info( @@ -165,146 +180,120 @@ def main(): ) sys.exit(0) - cache = rss2irc.read_cache(logger, args.cache) scrub_items(logger, cache) - - # Note: I have failed to find web link to repo in GH response. + # NOTE(zstyblik): I have failed to find web link to repo in GH response. # Therefore, let's create one. repository_url = get_gh_repository_url(args.gh_owner, args.gh_repo) item_expiration = int(time.time()) + args.cache_expiration to_publish = process_page_items( logger, cache, pages, item_expiration, repository_url ) - + gh_data = GHRepoInfo( + repo_owner=args.gh_owner, + repo_name=args.gh_repo, + repo_section=args.gh_section, + ) if not args.cache_init and to_publish: slack_client = rss2slack.get_slack_web_client( slack_token, args.slack_base_url, args.slack_timeout ) - for html_url in to_publish: - cache_item = cache.items[html_url] - try: - msg_blocks = [ - format_message( - logger, - args.gh_owner, - args.gh_repo, - ALIASES[args.gh_section], - html_url, - cache_item, - ) - ] - rss2slack.post_to_slack( - logger, - msg_blocks, - slack_client, - args.slack_channel, - ) - except Exception: - logger.error("%s", traceback.format_exc()) - cache.items.pop(html_url) - finally: - time.sleep(args.sleep) - - rss2irc.write_cache(cache, args.cache) + process_news( + logger, + cache, + to_publish, + args.sleep, + gh_data, + slack_client, + args.slack_channel, + ) + + retcode = 0 + except SlackTokenError: + logger.exception("Environment variable SLACK_TOKEN must be set.") + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) except Exception: - logger.debug("%s", traceback.format_exc()) - # TODO(zstyblik): - # 1. touch error file - # 2. send error message to the channel - finally: - sys.exit(0) + logger.exception("Unexpected exception has occurred.") + retcode = 1 + + write_retcode = rss2irc.wrap_write_cache(logger, cache, args.cache_file) + retcode = utils.escalate_retcode(write_retcode, retcode) + retcode = utils.mask_retcode(retcode, args.mask_errors) + sys.exit(retcode) def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( - "--cache", - dest="cache", - type=str, - default=None, - help="Path to cache file.", - ) - parser.add_argument( - "--cache-expiration", - dest="cache_expiration", - type=int, - default=config_options.CACHE_EXPIRATION, - help="Time, in seconds, for how long to keep items " "in cache.", - ) - parser.add_argument( - "--cache-init", - dest="cache_init", - action="store_true", - default=False, - help=( - "Prevents posting news to IRC. This is useful " - "when bootstrapping new RSS feed." - ), - ) - parser.add_argument( + cli_args.add_generic_args(parser) + cli_args.add_cache_file_arg_group(parser) + + github_group = parser.add_argument_group("GitHub options") + github_group.add_argument( "--gh-owner", dest="gh_owner", required=True, type=str, help="Owner/org of the repository to track.", ) - parser.add_argument( + github_group.add_argument( "--gh-repo", dest="gh_repo", required=True, type=str, help="Repository of owner/org to track.", ) - parser.add_argument( + github_group.add_argument( "--gh-section", dest="gh_section", required=True, choices=["issues", "pulls"], help='GH "section" to track.', ) - parser.add_argument( - "--slack-base-url", - dest="slack_base_url", - type=str, - default=rss2slack.SLACK_BASE_URL, - help="Base URL for Slack client.", - ) - parser.add_argument( - "--slack-channel", - dest="slack_channel", - type=str, - required=True, - help="Name of Slack channel to send formatted news to.", - ) - parser.add_argument( - "--slack-timeout", - dest="slack_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="Slack API Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--sleep", - dest="sleep", - type=int, - default=2, - help=( - "Sleep between messages in order to avoid " - "possible excess flood/API call rate limit." - ), - ) - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) - return parser.parse_args() + + cli_args.add_slack_arg_group(parser, rss2slack.SLACK_BASE_URL) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + cli_args.check_cache_expiration_arg(parser, args) + cli_args.check_sleep_arg(parser, args) + return args + + +def process_news( + logger: logging.Logger, + cache: CachedData, + to_publish: Set[str], + sleep: int, + gh_data: GHRepoInfo, + slack_client, + slack_channel: str, +): + """Process new items and post to Slack.""" + for html_url in to_publish: + cache_item = cache.items[html_url] + try: + msg_blocks = [ + format_message( + logger, + gh_data.repo_owner, + gh_data.repo_name, + ALIASES[gh_data.repo_section], + html_url, + cache_item, + ) + ] + rss2slack.post_to_slack( + logger, + msg_blocks, + slack_client, + slack_channel, + ) + except Exception: + logger.exception("Exception has occurred while posting to Slack") + cache.items.pop(html_url) + finally: + time.sleep(sleep) def process_page_items( @@ -360,8 +349,7 @@ def scrub_items(logger: logging.Logger, cache: CachedData) -> None: try: expiration = int(cache.items[key]["expiration"]) except (KeyError, ValueError): - logger.error("%s", traceback.format_exc()) - logger.error( + logger.exception( "Invalid cache entry will be removed: '%s'", cache.items[key] ) cache.items.pop(key) diff --git a/git_commits2slack.py b/git_commits2slack.py index 1fad68c..78d5d1e 100755 --- a/git_commits2slack.py +++ b/git_commits2slack.py @@ -12,12 +12,13 @@ import re import subprocess import sys -import traceback from typing import Dict from typing import List import rss2slack -from lib import config_options +from lib import cli_args +from lib import utils +from lib.exceptions import SlackTokenError RE_GIT_AUTD = re.compile(r"^Already up-to-date.$") RE_GIT_UPDATING = re.compile(r"^Updating [a-z0-9]+", re.I) @@ -156,12 +157,12 @@ def git_show(git_clone_dir: str, git_ref: str) -> List[str]: def main(): """Post new commits in given repository to Slack.""" + args = parse_args() logging.basicConfig(stream=sys.stdout, level=logging.ERROR) logger = logging.getLogger("git-commits2slack") - args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) + logger.setLevel(args.log_level) + retcode = 0 try: slack_token = rss2slack.get_slack_token() @@ -176,7 +177,6 @@ def main(): commits = git_show(args.git_clone_dir, commit_ref) if not commits: - # FIXME(zstyblik): error? send message to Slack? logger.warning("There should be new commits, but we have none.") sys.exit(0) @@ -203,81 +203,51 @@ def main(): slack_client, args.slack_channel, ) + except SlackTokenError: + logger.exception("Environment variable SLACK_TOKEN must be set.") + retcode = 1 except Exception: - logger.debug("%s", traceback.format_exc()) - # TODO(zstyblik): - # 1. touch error file - # 2. send error message to the channel - finally: - sys.exit(0) + logger.exception("Unexpected exception has occurred.") + retcode = 1 + + retcode = utils.mask_retcode(retcode, args.mask_errors) + sys.exit(retcode) def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( + cli_args.add_generic_args(parser) + + git_group = parser.add_argument_group("git options") + git_group.add_argument( "--git-clone-dir", dest="git_clone_dir", required=True, type=str, help="Directory where git repository will be cloned into.", ) - parser.add_argument( + git_group.add_argument( "--git-repository", dest="git_repo", required=True, type=str, help="git repository to track.", ) - parser.add_argument( + git_group.add_argument( "--git-web", dest="git_web", type=str, default="http://localhost", help="git web interface, resp. base URL, for given repository.", ) - parser.add_argument( - "--slack-base-url", - dest="slack_base_url", - type=str, - default=rss2slack.SLACK_BASE_URL, - help="Base URL for Slack client.", - ) - parser.add_argument( - "--slack-channel", - dest="slack_channel", - type=str, - required=True, - help="Name of Slack channel to send formatted news to.", - ) - parser.add_argument( - "--slack-timeout", - dest="slack_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="Slack API Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--sleep", - dest="sleep", - type=int, - default=2, - help=( - "Sleep between messages in order to avoid " - "possible excess flood/API call rate limit." - ), - ) - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) - return parser.parse_args() + + cli_args.add_slack_arg_group(parser, rss2slack.SLACK_BASE_URL) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + cli_args.check_sleep_arg(parser, args) + return args def parse_commits(output: str) -> List[str]: diff --git a/lib/cli_args.py b/lib/cli_args.py new file mode 100644 index 0000000..663893a --- /dev/null +++ b/lib/cli_args.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Helper functions related to CLI args. + +2026/Jan/06 @ Zdenek Styblik +""" +import argparse +from dataclasses import dataclass + +from . import config_options + + +@dataclass +class GenericArgsCfg: + """Class represents configuration of generic CLI args.""" + + handle: bool = False + output: bool = False + + +def add_cache_file_arg_group(parser: argparse.ArgumentParser): + """Add cache related CLI args.""" + cache_file_group = parser.add_argument_group("caching options") + cache_file_group.add_argument( + "--cache", + dest="cache_file", + type=str, + default=None, + help="File which contains cache.", + ) + cache_file_group.add_argument( + "--cache-expiration", + dest="cache_expiration", + type=int, + default=config_options.CACHE_EXPIRATION, + help=( + "How long to keep items in cache. " + "Defaults to %(default)s seconds." + ), + ) + cache_file_group.add_argument( + "--cache-init", + dest="cache_init", + action="store_true", + default=False, + help=( + "Prevents posting news to IRC. This is useful " + "when bootstrapping new RSS feed." + ), + ) + + +def add_generic_args( + parser: argparse.ArgumentParser, args_cfg: GenericArgsCfg = None +): + """Add generic CLI args.""" + if args_cfg and args_cfg.handle: + parser.add_argument( + "--handle", + dest="handle", + type=str, + default=None, + help="Handle/call sign of this feed.", + ) + + if args_cfg and args_cfg.output: + parser.add_argument( + "--output", + dest="output", + type=str, + required=True, + help="Where to output formatted news.", + ) + + parser.add_argument( + "--return-error", + dest="mask_errors", + action="store_false", + default=True, + help=( + "Return RC > 0 should error occur. " + "Majority of errors are masked because of cron." + ), + ) + parser.add_argument( + "--sleep", + dest="sleep", + type=int, + default=2, + help="Sleep between messages in order to avoid Excess Flood at IRC.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Increase log level verbosity. Can be passed multiple times.", + ) + + +def add_rss_arg_group(parser: argparse.ArgumentParser): + """Add RSS related CLI args.""" + rss_group = parser.add_argument_group("RSS options") + rss_group.add_argument( + "--rss-url", + dest="rss_url", + type=str, + required=True, + help="URL of RSS Feed.", + ) + rss_group.add_argument( + "--rss-http-timeout", + dest="rss_http_timeout", + type=int, + default=config_options.HTTP_TIMEOUT, + help="HTTP Timeout. Defaults to %(default)s seconds.", + ) + + +def add_slack_arg_group(parser: argparse.ArgumentParser, slack_base_url: str): + """Add Slack related CLI args.""" + slack_group = parser.add_argument_group("Slack options") + slack_group.add_argument( + "--slack-base-url", + dest="slack_base_url", + type=str, + default=slack_base_url, + help="Base URL for Slack client.", + ) + slack_group.add_argument( + "--slack-channel", + dest="slack_channel", + type=str, + required=True, + help="Name of Slack channel to send formatted news to.", + ) + slack_group.add_argument( + "--slack-timeout", + dest="slack_timeout", + type=int, + default=config_options.HTTP_TIMEOUT, + help="Slack API Timeout. Defaults to %(default)s seconds.", + ) + + +def check_cache_expiration_arg( + parser: argparse.ArgumentParser, args: argparse.Namespace +): + """Check that cache_expiration CLI arg is within range.""" + if args.cache_expiration < 0: + parser.error("Cache expiration cannot be less than 0.") + + +def check_sleep_arg(parser: argparse.ArgumentParser, args: argparse.Namespace): + """Check that sleep CLI arg is within range.""" + if args.sleep < 0: + parser.error("Sleep interval cannot be less than 0.") diff --git a/lib/exceptions.py b/lib/exceptions.py new file mode 100644 index 0000000..13f7b0e --- /dev/null +++ b/lib/exceptions.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Exceptions for RSS2IRC.""" + + +class RSS2IRCBaseException(Exception): + """RSS2IRC base exception.""" + + +class CacheReadError(RSS2IRCBaseException): + """Raised when an error occurs while reading the cache file.""" + + +class CacheWriteError(RSS2IRCBaseException): + """Raised when an error occurs while writing the cache file.""" + + +class EmptyResponseError(RSS2IRCBaseException): + """Raised when an empty HTTP response has been received.""" + + +class NoNewsError(RSS2IRCBaseException): + """Raised when RSS has no news.""" + + +class NotModifiedError(RSS2IRCBaseException): + """Raised when HTTP content has not changed.""" + + +class SlackTokenError(RSS2IRCBaseException): + """Raised when SLACK_TOKEN env variable is not set or empty.""" diff --git a/lib/http_source.py b/lib/http_source.py index 470c4ef..1aa59ef 100644 --- a/lib/http_source.py +++ b/lib/http_source.py @@ -13,6 +13,7 @@ class HTTPSource: """Class represents HTTP data source.""" + http_error_count: int = 0 http_etag: str = field(default_factory=str) http_last_modified: str = field(default_factory=str) last_used_ts: int = 0 diff --git a/lib/tests/test_http_source.py b/lib/tests/test_http_source.py index adfa261..c29f435 100644 --- a/lib/tests/test_http_source.py +++ b/lib/tests/test_http_source.py @@ -2,7 +2,7 @@ """Unit tests for http_source.py.""" import pytest -from lib import HTTPSource # noqa: I202 +from lib import HTTPSource @pytest.mark.parametrize( diff --git a/lib/tests/test_utils.py b/lib/tests/test_utils.py new file mode 100644 index 0000000..c0fb12c --- /dev/null +++ b/lib/tests/test_utils.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Unit tests for utils.py.""" +import pytest + +from lib import utils + + +@pytest.mark.parametrize( + "new_rc,old_rc,expected", + [ + (0, 0, 0), + (0, 1, 1), + (1, 1, 1), + (1, 0, 1), + (2, 1, 2), + (1, 2, 2), + (-1, 0, 0), + ], +) +def test_escalate_retcode(new_rc, old_rc, expected): + """Test that escalate_retcode() works as expected.""" + result = utils.escalate_retcode(new_rc, old_rc) + assert result == expected + + +@pytest.mark.parametrize( + "retcode,mask,expected", + [ + (0, False, 0), + (0, True, 0), + (1, False, 1), + (1, True, 0), + (20, False, 20), + (20, True, 0), + ], +) +def test_mask_retcode(retcode, mask, expected): + """Test that mask_retcode() works as expected.""" + result = utils.mask_retcode(retcode, mask) + assert result == expected diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..8a8be4b --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Utility functions used in rss2irc.""" + + +def calc_log_level(count: int) -> int: + """Return logging log level as int based on count.""" + log_level = 40 - max(count, 0) * 10 + log_level = max(log_level, 10) + return log_level + + +def escalate_retcode(new_rc: int, old_rc: int) -> int: + """Return retcode which is bigger.""" + retcode = old_rc + if new_rc > old_rc: + retcode = new_rc + + return retcode + + +def mask_retcode(retcode: int, mask: bool) -> int: + """Determine whether or not to mask error return code, if so mask it.""" + if mask: + retcode = 0 + + return retcode diff --git a/migrations/tests/test_convert_cache_to_dataclass_v1.py b/migrations/tests/test_convert_cache_to_dataclass_v1.py index 86606b3..72e3808 100644 --- a/migrations/tests/test_convert_cache_to_dataclass_v1.py +++ b/migrations/tests/test_convert_cache_to_dataclass_v1.py @@ -8,8 +8,8 @@ import pytest -import rss2irc # noqa: I202 -from lib import CachedData # noqa: I202 +import rss2irc +from lib import CachedData SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/phpbb2slack.py b/phpbb2slack.py index 07f26ae..9d67cdf 100755 --- a/phpbb2slack.py +++ b/phpbb2slack.py @@ -7,16 +7,21 @@ import logging import sys import time -import traceback from typing import Dict from typing import List import feedparser -import rss2irc # noqa: I202 -import rss2slack # noqa: I202 -from lib import CachedData # noqa: I202 -from lib import config_options # noqa: I202 +import rss2irc +import rss2slack +from lib import CachedData +from lib import cli_args +from lib import config_options +from lib import utils +from lib.exceptions import EmptyResponseError +from lib.exceptions import NoNewsError +from lib.exceptions import NotModifiedError +from lib.exceptions import SlackTokenError def format_message( @@ -59,7 +64,10 @@ def get_authors_from_file(logger: logging.Logger, fname: str) -> List[str]: if line.decode("utf-8").strip() != "" ] except Exception: - logger.error("%s", traceback.format_exc()) + logger.exception( + "Failed to parse authors from file '%s' due to exception.", + fname, + ) authors = [] return authors @@ -67,21 +75,21 @@ def get_authors_from_file(logger: logging.Logger, fname: str) -> List[str]: def main(): """Fetch phpBB RSS feed and post RSS news to Slack.""" + args = parse_args() logging.basicConfig(stream=sys.stdout, level=logging.ERROR) logger = logging.getLogger("phpbb2slack") - args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) + logger.setLevel(args.log_level) - if args.cache_expiration < 0: - logger.error("Cache expiration can't be less than 0.") - sys.exit(1) + retcode = 0 + cache = rss2irc.wrap_read_cache(logger, args.cache_file) + if cache is None: + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) + source = cache.get_source_by_url(args.rss_url) try: slack_token = rss2slack.get_slack_token() authors = get_authors_from_file(logger, args.authors_file) - cache = rss2irc.read_cache(logger, args.cache) - source = cache.get_source_by_url(args.rss_url) rsp = rss2irc.get_rss( logger, @@ -89,58 +97,71 @@ def main(): args.rss_http_timeout, source.make_caching_headers(), ) - if rsp.status_code == 304: - logger.debug("No new RSS data since the last run") - rss2irc.write_cache(cache, args.cache) - sys.exit(0) - - if not rsp.text: - logger.error("Failed to get RSS from %s", args.rss_url) - sys.exit(1) - news = parse_news(rsp.text, authors) if not news: - logger.info("No news?") - sys.exit(0) + raise NoNewsError source.extract_caching_headers(rsp.headers) prune_news(logger, cache, news, args.cache_expiration) scrub_items(logger, cache) - slack_client = rss2slack.get_slack_web_client( - slack_token, args.slack_base_url, args.slack_timeout - ) if not args.cache_init: - for url in list(news.keys()): - msg_blocks = [format_message(url, news[url], args.handle)] - try: - rss2slack.post_to_slack( - logger, - msg_blocks, - slack_client, - args.slack_channel, - ) - except ValueError: - news.pop(url) - finally: - time.sleep(args.sleep) + slack_client = rss2slack.get_slack_web_client( + slack_token, args.slack_base_url, args.slack_timeout + ) + process_news( + logger, + news, + args.handle, + args.sleep, + slack_client, + args.slack_channel, + ) update_items_expiration(cache, news, args.cache_expiration) cache.scrub_data_sources() - rss2irc.write_cache(cache, args.cache) + source.http_error_count = 0 + retcode = 0 + except SlackTokenError: + logger.exception("Environment variable SLACK_TOKEN must be set.") + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) + except NotModifiedError: + logger.debug("No new RSS data since the last run.") + update_items_expiration(cache, cache.items, args.cache_expiration) + source.http_error_count = 0 + retcode = 0 + except EmptyResponseError: + logger.error("Got empty response from '%s'.", args.rss_url) + source.http_error_count += 1 + retcode = 1 + except NoNewsError: + # NOTE(zstyblik): some feeds don't have news unless something is up, eg. + # AWS RSS feed doesn't have news unless there is a problem. + logger.info("No news from '%s'?", args.rss_url) + update_items_expiration(cache, cache.items, args.cache_expiration) + # NOTE(zstyblik): leave source.http_error_count unchanged + retcode = 0 except Exception: - logger.debug("%s", traceback.format_exc()) - # TODO(zstyblik): - # 1. touch error file - # 2. send error message to the channel - finally: - sys.exit(0) + logger.exception("Unexpected exception has occurred.") + source.http_error_count += 1 + retcode = 1 + + write_retcode = rss2irc.wrap_write_cache(logger, cache, args.cache_file) + retcode = utils.escalate_retcode(write_retcode, retcode) + retcode = utils.mask_retcode(retcode, args.mask_errors) + sys.exit(retcode) def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( + generic_args = cli_args.GenericArgsCfg(handle=True) + cli_args.add_generic_args(parser, generic_args) + cli_args.add_cache_file_arg_group(parser) + + phpbb_group = parser.add_argument_group("phpBB options") + phpbb_group.add_argument( "--authors-of-interest", dest="authors_file", type=str, @@ -151,95 +172,15 @@ def parse_args() -> argparse.Namespace: "list will be pushed." ), ) - parser.add_argument( - "--cache", - dest="cache", - type=str, - default=None, - help="Path to cache file.", - ) - parser.add_argument( - "--cache-expiration", - dest="cache_expiration", - type=int, - default=config_options.CACHE_EXPIRATION, - help="Time, in seconds, for how long to keep items in cache.", - ) - parser.add_argument( - "--cache-init", - dest="cache_init", - action="store_true", - default=False, - help=( - "Prevents posting news to IRC. This is useful " - "when bootstrapping new RSS feed." - ), - ) - parser.add_argument( - "--handle", - dest="handle", - type=str, - default=None, - help="Handle/callsign of this feed.", - ) - parser.add_argument( - "--rss-url", - dest="rss_url", - type=str, - required=True, - help="URL of RSS Feed.", - ) - parser.add_argument( - "--rss-http-timeout", - dest="rss_http_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="HTTP Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--slack-base-url", - dest="slack_base_url", - type=str, - default=rss2slack.SLACK_BASE_URL, - help="Base URL for Slack client.", - ) - parser.add_argument( - "--slack-channel", - dest="slack_channel", - type=str, - required=True, - help="Name of Slack channel to send formatted news to.", - ) - parser.add_argument( - "--slack-timeout", - dest="slack_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="Slack API Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--sleep", - dest="sleep", - type=int, - default=2, - help=( - "Sleep between messages in order to avoid " - "possible excess flood/API call rate limit." - ), - ) - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) - return parser.parse_args() + + cli_args.add_rss_arg_group(parser) + cli_args.add_slack_arg_group(parser, rss2slack.SLACK_BASE_URL) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + cli_args.check_cache_expiration_arg(parser, args) + cli_args.check_sleep_arg(parser, args) + return args def parse_news(data: str, authors: List[str]) -> Dict: @@ -273,6 +214,30 @@ def parse_news(data: str, authors: List[str]) -> Dict: return news +def process_news( + logger: logging.Logger, + news, + handle: str, + sleep: int, + slack_client, + slack_channel: str, +): + """Process news and post it to Slack.""" + for url in list(news.keys()): + msg_blocks = [format_message(url, news[url], handle)] + try: + rss2slack.post_to_slack( + logger, + msg_blocks, + slack_client, + slack_channel, + ) + except ValueError: + news.pop(url) + finally: + time.sleep(sleep) + + def prune_news( logger: logging.Logger, cache: CachedData, @@ -300,8 +265,7 @@ def scrub_items(logger: logging.Logger, cache: CachedData) -> None: try: expiration = int(cache.items[key]["expiration"]) except (KeyError, ValueError): - logger.error("%s", traceback.format_exc()) - logger.error( + logger.exception( "Invalid cache entry will be removed: '%s'", cache.items[key] ) cache.items.pop(key) diff --git a/rss2irc.py b/rss2irc.py index 516a170..1f00aec 100755 --- a/rss2irc.py +++ b/rss2irc.py @@ -19,8 +19,15 @@ import feedparser import requests -from lib import CachedData # noqa: I202 -from lib import config_options # noqa: I202 +from lib import CachedData +from lib import cli_args +from lib import config_options +from lib import utils +from lib.exceptions import CacheReadError +from lib.exceptions import CacheWriteError +from lib.exceptions import EmptyResponseError +from lib.exceptions import NoNewsError +from lib.exceptions import NotModifiedError def format_message( @@ -49,61 +56,67 @@ def get_rss( timeout: int = config_options.HTTP_TIMEOUT, extra_headers: Dict = None, ) -> requests.models.Response: - """Return body of given URL as a string.""" + """Return body of given URL as a string. + + :raises EmptyResponseError: raised when HTTP rsp body is empty + :raises NotModifiedError: raised when HTTP Status Code is 304 + :raises requests.exceptions.BaseHTTPError: raised when HTTP error occurs + """ # Randomize user agent, because CF likes to block for no apparent reason. - user_agent = "rss2irc_{:d}".format(int(time.time())) - headers = {"User-Agent": user_agent} + user_agent = "rss2irc-script" + headers = { + "User-Agent": user_agent, + "Accept": ( + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + ), + } if extra_headers: for key, value in extra_headers.items(): headers[key] = value - logger.debug("Get %s", url) + logger.debug("Make request at URL '%s'.", url) rsp = requests.get(url, timeout=timeout, headers=headers) - logger.debug("Got HTTP Status Code: %i", rsp.status_code) + logger.debug("Got HTTP Status Code '%i'.", rsp.status_code) rsp.raise_for_status() + + if rsp.status_code == 304: + raise NotModifiedError + + if not rsp.text: + raise EmptyResponseError + return rsp def main(): """Fetch RSS feed and post RSS news to IRC.""" - logging.basicConfig(stream=sys.stdout) - logger = logging.getLogger("rss2irc") args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) - - if args.cache_expiration < 0: - logger.error("Cache expiration can't be less than 0.") - sys.exit(1) + logging.basicConfig(level=args.log_level, stream=sys.stdout) + logger = logging.getLogger("rss2irc") + logger.setLevel(args.log_level) if not os.path.exists(args.output): logger.error("Ouput '%s' doesn't exist.", args.output) sys.exit(1) - try: - cache = read_cache(logger, args.cache) - source = cache.get_source_by_url(args.rss_url) + retcode = 0 + cache = wrap_read_cache(logger, args.cache_file) + if cache is None: + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) + source = cache.get_source_by_url(args.rss_url) + try: rsp = get_rss( logger, args.rss_url, args.rss_http_timeout, source.make_caching_headers(), ) - if rsp.status_code == 304: - logger.debug("No new RSS data since the last run") - write_cache(cache, args.cache) - sys.exit(0) - - if not rsp.text: - logger.error("Failed to get RSS from %s", args.rss_url) - sys.exit(1) news = parse_news(rsp.text) if not news: - logger.info("No news?") - write_cache(cache, args.cache) - sys.exit(0) + raise NoNewsError source.extract_caching_headers(rsp.headers) prune_news(logger, cache, news, args.cache_expiration) @@ -114,90 +127,48 @@ def main(): update_items_expiration(cache, news, args.cache_expiration) cache.scrub_data_sources() - write_cache(cache, args.cache) - # TODO(zstyblik): remove error file + source.http_error_count = 0 + retcode = 0 + except NotModifiedError: + logger.debug("No new RSS data since the last run.") + update_items_expiration(cache, cache.items, args.cache_expiration) + source.http_error_count = 0 + retcode = 0 + except EmptyResponseError: + logger.error("Got empty response from '%s'.", args.rss_url) + source.http_error_count += 1 + retcode = 1 + except NoNewsError: + # NOTE(zstyblik): some feeds don't have news unless something is up, eg. + # AWS RSS feed doesn't have news unless there is a problem. + logger.info("No news from '%s'?", args.rss_url) + update_items_expiration(cache, cache.items, args.cache_expiration) + # NOTE(zstyblik): leave source.http_error_count unchanged + retcode = 0 except Exception: - logger.debug("%s", traceback.format_exc()) - # TODO(zstyblik): - # 1. touch error file - # 2. send error message to the channel - finally: - sys.exit(0) + logger.exception("Unexpected exception has occurred.") + source.http_error_count += 1 + retcode = 1 + + write_retcode = wrap_write_cache(logger, cache, args.cache_file) + retcode = utils.escalate_retcode(write_retcode, retcode) + retcode = utils.mask_retcode(retcode, args.mask_errors) + sys.exit(retcode) def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) - parser.add_argument( - "--rss-url", - dest="rss_url", - type=str, - required=True, - help="URL of RSS Feed.", - ) - parser.add_argument( - "--rss-http-timeout", - dest="rss_http_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="HTTP Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--handle", - dest="handle", - type=str, - default=None, - help="IRC handle of this feed.", - ) - parser.add_argument( - "--output", - dest="output", - type=str, - required=True, - help="Where to output formatted news.", - ) - parser.add_argument( - "--cache", - dest="cache", - type=str, - default=None, - help="File which contains cache.", - ) - parser.add_argument( - "--cache-expiration", - dest="cache_expiration", - type=int, - default=config_options.CACHE_EXPIRATION, - help="Time, in seconds, for how long to keep items in cache.", - ) - parser.add_argument( - "--cache-init", - dest="cache_init", - action="store_true", - default=False, - help=( - "Prevents posting news to IRC. This is useful " - "when bootstrapping new RSS feed." - ), - ) - parser.add_argument( - "--sleep", - dest="sleep", - type=int, - default=2, - help="Sleep between messages in order to avoid Excess Flood at IRC.", - ) - return parser.parse_args() + generic_args = cli_args.GenericArgsCfg(handle=True, output=True) + cli_args.add_generic_args(parser, generic_args) + cli_args.add_cache_file_arg_group(parser) + cli_args.add_rss_arg_group(parser) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + cli_args.check_cache_expiration_arg(parser, args) + cli_args.check_sleep_arg(parser, args) + return args def parse_news(data: str) -> Dict[str, Tuple[str, str]]: @@ -233,7 +204,10 @@ def prune_news( def read_cache(logger: logging.Logger, cache_file: str) -> CachedData: - """Read file with Py pickle in it.""" + """Read file with Py pickle in it. + + :raises CacheReadError: raised when unhandled exception occurs + """ if not cache_file: return CachedData() @@ -244,23 +218,20 @@ def read_cache(logger: logging.Logger, cache_file: str) -> CachedData: cache = CachedData() logger.warning("Cache file '%s' doesn't exist.", cache_file) except EOFError: - # Note: occurred with empty file. + # NOTE(zstyblik): occurred with empty file. cache = CachedData() logger.debug( "Cache file '%s' is probably empty: %s", cache_file, traceback.format_exc(), ) + except Exception as exception: + raise CacheReadError from exception - logger.debug("%s", cache) + logger.debug("Cache: %s", cache) return cache -def signal_handler(signum, frame): - """Handle SIGALRM signal.""" - raise TimeoutError - - def scrub_items(logger: logging.Logger, cache: CachedData) -> None: """Scrub cache and remove expired items.""" time_now = time.time() @@ -268,8 +239,7 @@ def scrub_items(logger: logging.Logger, cache: CachedData) -> None: try: expiration = int(cache.items[key]) except ValueError: - logger.error("%s", traceback.format_exc()) - logger.error( + logger.exception( "Invalid cache entry will be removed: '%s'", cache.items[key] ) cache.items.pop(key) @@ -280,6 +250,11 @@ def scrub_items(logger: logging.Logger, cache: CachedData) -> None: cache.items.pop(key) +def signal_handler(signum, frame): + """Handle SIGALRM signal.""" + raise TimeoutError + + def update_items_expiration( cache: CachedData, news: Dict[str, Tuple[str, str]], @@ -291,13 +266,50 @@ def update_items_expiration( cache.items[key] = item_expiration -def write_cache(data: CachedData, cache_file: str) -> None: - """Dump data into file as a pickle.""" - if not cache_file: +def wrap_read_cache(logger: logging.Logger, cache_file: str): + """Call read_cache() and return cached data or log error and return None.""" + cache = None + try: + cache = read_cache(logger, cache_file) + except CacheReadError: + logger.exception("Error while reading cache file '%s'.", cache_file) + cache = None + + return cache + + +def wrap_write_cache( + logger: logging.Logger, + cache: CachedData, + cache_file: str, +) -> int: + """Call write_cache() and return 0 on success or 1 on error.""" + retcode = 0 + try: + write_cache(cache, cache_file) + except CacheWriteError: + logger.exception( + "Failed to write data into cache file '%s'.", + cache_file, + ) + retcode = 1 + + return retcode + + +def write_cache(data: CachedData, cache_file: str): + """Dump data into file as a pickle. + + :raises CacheWriteError: raised when unhandled exception occurs + """ + if not cache_file or data is None: return - with open(cache_file, "wb") as fhandle: - pickle.dump(data, fhandle, pickle.HIGHEST_PROTOCOL) + try: + with open(cache_file, "wb") as fhandle: + pickle.dump(data, fhandle, pickle.HIGHEST_PROTOCOL) + except Exception as exception: + raise CacheWriteError from exception def write_data( @@ -318,8 +330,12 @@ def write_data( write_message(logger, fhandle, message) time.sleep(sleep) except (TimeoutError, ValueError): - logger.debug("%s", traceback.format_exc()) - logger.debug("Failed to write %s, %s", url, data[url]) + logger.debug( + "Failed to write '%s'=>'%s' due to exception: %s", + url, + data[url], + traceback.format_exc(), + ) data.pop(url) diff --git a/rss2slack.py b/rss2slack.py index 4033c08..4d39833 100755 --- a/rss2slack.py +++ b/rss2slack.py @@ -15,8 +15,13 @@ from slack import WebClient -import rss2irc # noqa: I100, I202 -from lib import config_options # noqa: I100, I202 +import rss2irc +from lib import cli_args +from lib import utils +from lib.exceptions import EmptyResponseError +from lib.exceptions import NoNewsError +from lib.exceptions import NotModifiedError +from lib.exceptions import SlackTokenError SLACK_BASE_URL = WebClient.BASE_URL @@ -51,13 +56,13 @@ def format_message( def get_slack_token() -> str: """Get Slack token from ENV variable. - :raises: `ValueError` + :raises SlackTokenError: raised when env variable SLACK_TOKEN is not set """ slack_token = os.environ.get("SLACK_TOKEN", None) if slack_token: return slack_token - raise ValueError("SLACK_TOKEN must be set.") + raise SlackTokenError("SLACK_TOKEN env variable must be set") def get_slack_web_client(token: str, base_url: str, timeout: int) -> WebClient: @@ -67,20 +72,20 @@ def get_slack_web_client(token: str, base_url: str, timeout: int) -> WebClient: def main(): """Fetch RSS feed and post RSS news to Slack.""" - logging.basicConfig(stream=sys.stdout, level=logging.ERROR) - logger = logging.getLogger("rss2slack") args = parse_args() - if args.verbosity: - logger.setLevel(logging.DEBUG) + logging.basicConfig(level=args.log_level, stream=sys.stdout) + logger = logging.getLogger("rss2slack") + logger.setLevel(args.log_level) - if args.cache_expiration < 0: - logger.error("Cache expiration can't be less than 0.") - sys.exit(1) + retcode = 0 + cache = rss2irc.wrap_read_cache(logger, args.cache_file) + if cache is None: + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) + source = cache.get_source_by_url(args.rss_url) try: slack_token = get_slack_token() - cache = rss2irc.read_cache(logger, args.cache) - source = cache.get_source_by_url(args.rss_url) rsp = rss2irc.get_rss( logger, @@ -88,149 +93,82 @@ def main(): args.rss_http_timeout, source.make_caching_headers(), ) - if rsp.status_code == 304: - logger.debug("No new RSS data since the last run") - rss2irc.write_cache(cache, args.cache) - sys.exit(0) - - if not rsp.text: - logger.error("Failed to get RSS from %s", args.rss_url) - sys.exit(1) - news = rss2irc.parse_news(rsp.text) if not news: - logger.info("No news?") - sys.exit(0) + raise NoNewsError source.extract_caching_headers(rsp.headers) rss2irc.prune_news(logger, cache, news, args.cache_expiration) rss2irc.scrub_items(logger, cache) - slack_client = get_slack_web_client( - slack_token, - base_url=args.slack_base_url, - timeout=args.slack_timeout, - ) if not args.cache_init: - for url in list(news.keys()): - msg_blocks = [format_message(url, news[url], args.handle)] - try: - post_to_slack( - logger, - msg_blocks, - slack_client, - args.slack_channel, - ) - except ValueError: - news.pop(url) - finally: - time.sleep(args.sleep) + slack_client = get_slack_web_client( + slack_token, + base_url=args.slack_base_url, + timeout=args.slack_timeout, + ) + process_news( + logger, + news, + args.handle, + args.sleep, + slack_client, + args.slack_channel, + ) rss2irc.update_items_expiration(cache, news, args.cache_expiration) cache.scrub_data_sources() - rss2irc.write_cache(cache, args.cache) - # TODO(zstyblik): remove error file + source.http_error_count = 0 + retcode = 0 + except SlackTokenError: + logger.exception("Environment variable SLACK_TOKEN must be set.") + retcode = utils.mask_retcode(1, args.mask_errors) + sys.exit(retcode) + except NotModifiedError: + logger.debug("No new RSS data since the last run.") + rss2irc.update_items_expiration( + cache, cache.items, args.cache_expiration + ) + source.http_error_count = 0 + retcode = 0 + except EmptyResponseError: + logger.error("Got empty response from '%s'.", args.rss_url) + source.http_error_count += 1 + retcode = 1 + except NoNewsError: + # NOTE(zstyblik): some feeds don't have news unless something is up, eg. + # AWS RSS feed doesn't have news unless there is a problem. + logger.info("No news from '%s'?", args.rss_url) + rss2irc.update_items_expiration( + cache, cache.items, args.cache_expiration + ) + # NOTE(zstyblik): leave source.http_error_count unchanged + retcode = 0 except Exception: - logger.debug("%s", traceback.format_exc()) - # TODO(zstyblik): - # 1. touch error file - # 2. send error message to the channel - finally: - sys.exit(0) + logger.exception("Unexpected exception has occurred.") + source.http_error_count += 1 + retcode = 1 + + write_retcode = rss2irc.wrap_write_cache(logger, cache, args.cache_file) + retcode = utils.escalate_retcode(write_retcode, retcode) + retcode = utils.mask_retcode(retcode, args.mask_errors) + sys.exit(retcode) def parse_args() -> argparse.Namespace: """Return parsed CLI args.""" parser = argparse.ArgumentParser() - parser.add_argument( - "--cache", - dest="cache", - type=str, - default=None, - help="File which contains cache.", - ) - parser.add_argument( - "--cache-expiration", - dest="cache_expiration", - type=int, - default=config_options.CACHE_EXPIRATION, - help="Time, in seconds, for how long to keep items in cache.", - ) - parser.add_argument( - "--cache-init", - dest="cache_init", - action="store_true", - default=False, - help=( - "Prevents posting news to IRC. This is useful " - "when bootstrapping new RSS feed." - ), - ) - parser.add_argument( - "--handle", - dest="handle", - type=str, - default=None, - help="Handle/callsign of this feed.", - ) - parser.add_argument( - "--rss-url", - dest="rss_url", - type=str, - required=True, - help="URL of RSS Feed.", - ) - parser.add_argument( - "--rss-http-timeout", - dest="rss_http_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="HTTP Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--slack-base-url", - dest="slack_base_url", - type=str, - default=SLACK_BASE_URL, - help="Base URL for Slack client.", - ) - parser.add_argument( - "--slack-channel", - dest="slack_channel", - type=str, - required=True, - help="Name of Slack channel to send formatted news to.", - ) - parser.add_argument( - "--slack-timeout", - dest="slack_timeout", - type=int, - default=config_options.HTTP_TIMEOUT, - help="Slack API Timeout. Defaults to {:d} seconds.".format( - config_options.HTTP_TIMEOUT - ), - ) - parser.add_argument( - "--sleep", - dest="sleep", - type=int, - default=2, - help=( - "Sleep between messages in order to avoid " - "possible excess flood/API call rate limit." - ), - ) - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="store_true", - default=False, - help="Increase logging verbosity.", - ) - return parser.parse_args() + generic_args = cli_args.GenericArgsCfg(handle=True) + cli_args.add_generic_args(parser, generic_args) + cli_args.add_cache_file_arg_group(parser) + cli_args.add_rss_arg_group(parser) + cli_args.add_slack_arg_group(parser, SLACK_BASE_URL) + args = parser.parse_args() + args.log_level = utils.calc_log_level(args.verbose) + + cli_args.check_cache_expiration_arg(parser, args) + cli_args.check_sleep_arg(parser, args) + return args def post_to_slack( @@ -246,12 +184,43 @@ def post_to_slack( channel=slack_channel, blocks=msg_blocks ) logger.debug("Response from Slack: %s", rsp) - if not rsp or rsp["ok"] is False: + if not rsp: + raise ValueError("Slack response is not OK.") + + is_ok = rsp.get("ok", False) + if not is_ok: raise ValueError("Slack response is not OK.") except ValueError: - logger.debug("%s", traceback.format_exc()) + logger.debug( + "Failed to post to Slack due to exception: %s", + traceback.format_exc(), + ) raise +def process_news( + logger: logging.Logger, + news: Dict, + handle: str, + sleep: int, + slack_client: WebClient, + slack_channel: str, +) -> None: + """Process news and post it to Slack.""" + for url in list(news.keys()): + msg_blocks = [format_message(url, news[url], handle)] + try: + post_to_slack( + logger, + msg_blocks, + slack_client, + slack_channel, + ) + except ValueError: + news.pop(url) + finally: + time.sleep(sleep) + + if __name__ == "__main__": main() diff --git a/tests/files/rss_no_news.xml b/tests/files/rss_no_news.xml new file mode 100644 index 0000000..7bd062f --- /dev/null +++ b/tests/files/rss_no_news.xml @@ -0,0 +1,9 @@ + + + + Test Site - no news + https://www.example.com/ + en-us + RSS without any news. + + diff --git a/tests/test_cache_stats.py b/tests/test_cache_stats.py index 41d68fe..463fc82 100644 --- a/tests/test_cache_stats.py +++ b/tests/test_cache_stats.py @@ -6,9 +6,9 @@ import time from unittest.mock import patch -import cache_stats # noqa: I202 -import rss2irc # noqa: I202 -from lib import CachedData # noqa: I202 +import cache_stats +import rss2irc +from lib import CachedData SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/test_gh2slack.py b/tests/test_gh2slack.py index db87e28..7f3933b 100644 --- a/tests/test_gh2slack.py +++ b/tests/test_gh2slack.py @@ -12,9 +12,11 @@ import pytest -import gh2slack # noqa: I100, I202 -import rss2irc # noqa: I100, I202 -from lib import CachedData # noqa: I100, I202 +import gh2slack +import rss2irc +from lib import CachedData +from lib.exceptions import CacheReadError +from lib.exceptions import SlackTokenError SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -22,7 +24,8 @@ class MockedResponse: """Mocked `requests.Response`.""" - def __init__(self, response, headers=None): # noqa: D107 + def __init__(self, response, headers=None): + """Init.""" self.response = response if headers: self.headers = headers @@ -100,12 +103,10 @@ def test_gh_request(mock_get): assert mocked_response._raise_for_status_called is True -@patch("gh2slack.time.time") @patch("requests.get") -def test_gh_request_follows_link_header(mock_get, mock_time): +def test_gh_request_follows_link_header(mock_get): """Test gh_request() follows up on 'Link' header.""" url = "https://api.github.com/repos/foo/bar" - mock_time.return_value = 123 mocked_response1 = MockedResponse( "foo", {"link": '; rel="next"'} ) @@ -116,7 +117,7 @@ def test_gh_request_follows_link_header(mock_get, mock_time): "https://api.github.com/repos/foo/bar", headers={ "Accept": "application/vnd.github.v3+json", - "User-Agent": "gh2slack_123", + "User-Agent": "gh2slack-script", }, params={"sort": "created", "state": "open"}, timeout=30, @@ -125,7 +126,7 @@ def test_gh_request_follows_link_header(mock_get, mock_time): "http://example.com", headers={ "Accept": "application/vnd.github.v3+json", - "User-Agent": "gh2slack_123", + "User-Agent": "gh2slack-script", }, params={"sort": "created", "state": "open"}, timeout=30, @@ -341,6 +342,317 @@ def test_main_ideal( assert data in expected_slack_requests +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("gh2slack.rss2irc.wrap_write_cache") +@patch("gh2slack.rss2irc.read_cache") +def test_main_cache_read_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that CacheReadError is handled as expected.""" + expected_log_records = [ + ( + "gh2slack", + 40, + "Error while reading cache file '/path/does/not/exist'.", + ), + ] + expected_slack_channel = "test" + fixture_cache_file = "/path/does/not/exist" + gh_repo = "test-repo" + gh_owner = "test-user" + gh_section = "pulls" + slack_base_url = "https://slack.example.com" + + mock_read_cache.side_effect = CacheReadError("pytest") + # + exception = None + args = [ + "./gh2slack.py", + "--cache", + fixture_cache_file, + "--gh-owner", + gh_owner, + "--gh-repo", + gh_repo, + "--gh-section", + gh_section, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("Slack URL: {:s}".format(slack_base_url)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + gh2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("gh2slack.rss2irc.wrap_write_cache") +@patch("gh2slack.rss2slack.get_slack_token") +@patch("gh2slack.rss2irc.read_cache") +def test_main_slack_token_error( + mock_read_cache, + mock_get_slack_token, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that SlackTokenError is handled as expected.""" + expected_log_records = [ + ( + "gh2slack", + 40, + "Environment variable SLACK_TOKEN must be set.", + ), + ] + expected_slack_channel = "test" + fixture_cache_file = "/path/does/not/exist" + gh_repo = "test-repo" + gh_owner = "test-user" + gh_section = "pulls" + slack_base_url = "https://slack.example.com" + + mock_read_cache.return_value = CachedData() + mock_get_slack_token.side_effect = SlackTokenError("pytest") + # + exception = None + args = [ + "./gh2slack.py", + "--cache", + fixture_cache_file, + "--gh-owner", + gh_owner, + "--gh-repo", + gh_repo, + "--gh-section", + gh_section, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("Slack URL: {:s}".format(slack_base_url)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + gh2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_slack_token.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("gh2slack.rss2irc.wrap_write_cache") +@patch("gh2slack.rss2slack.get_slack_token") +@patch("gh2slack.rss2irc.read_cache") +def test_main_random_exception( + mock_read_cache, + mock_get_slack_token, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that unexpected exception is handled correctly.""" + expected_log_records = [ + ( + "gh2slack", + 40, + "Unexpected exception has occurred.", + ), + ] + expected_slack_channel = "test" + fixture_cache_file = "/path/does/not/exist" + gh_repo = "test-repo" + gh_owner = "test-user" + gh_section = "pulls" + slack_base_url = "https://slack.example.com" + + mock_read_cache.return_value = CachedData() + mock_get_slack_token.side_effect = ValueError("pytest") + mock_wrap_write_cache.return_value = 0 + # + exception = None + args = [ + "./gh2slack.py", + "--cache", + fixture_cache_file, + "--gh-owner", + gh_owner, + "--gh-repo", + gh_repo, + "--gh-section", + gh_section, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("Slack URL: {:s}".format(slack_base_url)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + gh2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_slack_token.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("gh2slack.rss2irc.wrap_write_cache") +@patch("gh2slack.rss2irc.read_cache") +def test_main_wrap_write_cache_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + monkeypatch, + fixture_mock_requests, + caplog, +): + """Test that error in wrap_write_cache is handled as expected.""" + expected_log_records = [] + expected_slack_channel = "test" + expected_cache_keys = [ + "http://example.com/foo", + "http://example.com/bar", + ] + fixture_cache_file = "/path/does/not/exist" + gh_repo = "test-repo" + gh_owner = "test-user" + gh_section = "pulls" + gh_url = gh2slack.get_gh_api_url(gh_owner, gh_repo, gh_section) + slack_base_url = "https://slack.example.com" + + cache = CachedData() + mock_read_cache.return_value = cache + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + # Mock HTTP RSS + pages = [ + { + "html_url": "http://example.com/foo", + "number": 0, + "title": "some title#1", + }, + { + "html_url": "http://example.com/bar", + "number": 1, + "title": "some title#2", + }, + ] + mock_http_rss = fixture_mock_requests.get(gh_url, text=json.dumps(pages)) + mock_wrap_write_cache.return_value = 1 + # + exception = None + args = [ + "./gh2slack.py", + "--cache", + fixture_cache_file, + "--cache-init", + "--gh-owner", + gh_owner, + "--gh-repo", + gh_repo, + "--gh-section", + gh_section, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("Slack URL: {:s}".format(slack_base_url)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + gh2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + # Check HTTP RSS mock + assert mock_http_rss.called is True + assert mock_http_rss.call_count == 1 + assert mock_http_rss.last_request.text is None + + assert caplog.record_tuples == expected_log_records + assert list(cache.items.keys()) == expected_cache_keys + + def test_process_page_items(): """Test process_page_items().""" pages = [ diff --git a/tests/test_git_commits2slack.py b/tests/test_git_commits2slack.py index f6d7dd8..5bd8856 100644 --- a/tests/test_git_commits2slack.py +++ b/tests/test_git_commits2slack.py @@ -9,7 +9,8 @@ import pytest -import git_commits2slack # noqa:I100,I202 +import git_commits2slack +from lib.exceptions import SlackTokenError @pytest.fixture @@ -387,3 +388,117 @@ def test_main_ideal( assert req0.method == "POST" data = req0.get_json() assert data == expected_slack_requests[0] + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("git_commits2slack.rss2slack.get_slack_token") +def test_main_slack_token_error( + mock_get_slack_token, + extra_args, + expected_retcode, + fixture_git_dir, + caplog, +): + """Test that SlackTokenError is handled as expected.""" + expected_log_records = [ + ( + "git-commits2slack", + 40, + "Environment variable SLACK_TOKEN must be set.", + ), + ] + expected_slack_channel = "test" + expected_slack_url = "https://slack.example.com" + + mock_get_slack_token.side_effect = SlackTokenError("pytest") + + exception = None + args = [ + "./git_commits2slack.py", + "--git-clone-dir", + fixture_git_dir, + "--git-repository", + "test", + "--git-web", + "http://example.com", + "--slack-base-url", + expected_slack_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + with patch.object(sys, "argv", args): + try: + git_commits2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("git_commits2slack.rss2slack.get_slack_token") +def test_main_random_exception( + mock_get_slack_token, + extra_args, + expected_retcode, + fixture_git_dir, + caplog, +): + """Test that unexpected exception is handled as expected.""" + expected_log_records = [ + ( + "git-commits2slack", + 40, + "Unexpected exception has occurred.", + ), + ] + expected_slack_channel = "test" + expected_slack_url = "https://slack.example.com" + + mock_get_slack_token.side_effect = ValueError("pytest") + + exception = None + args = [ + "./git_commits2slack.py", + "--git-clone-dir", + fixture_git_dir, + "--git-repository", + "test", + "--git-web", + "http://example.com", + "--slack-base-url", + expected_slack_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + with patch.object(sys, "argv", args): + try: + git_commits2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + assert caplog.record_tuples == expected_log_records diff --git a/tests/test_phpbb2slack.py b/tests/test_phpbb2slack.py index 1f92353..1851a1a 100644 --- a/tests/test_phpbb2slack.py +++ b/tests/test_phpbb2slack.py @@ -9,10 +9,12 @@ import pytest -import phpbb2slack # noqa: I100, I202 -import rss2irc # noqa: I100, I202 -from lib import CachedData # noqa: I100, I202 -from lib import config_options # noqa: I100, I202 +import phpbb2slack +import rss2irc +from lib import CachedData +from lib import config_options +from lib.exceptions import CacheReadError +from lib.exceptions import EmptyResponseError ITEM_EXPIRATION = int(time.time()) SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -114,6 +116,7 @@ def test_main_ideal( handle = "test" http_timeout = "10" rss_url = "http://rss.example.com" + current_time = int(time.time()) expected_cache_keys = [ "https://phpbb.example.com/threads/something-of-something.424837/", ] @@ -161,10 +164,10 @@ def test_main_ideal( source1 = cache.get_source_by_url(rss_url) source1.http_etag = "" source1.http_last_modified = "" - source1.last_used_ts = int(time.time()) - 2 * 86400 + source1.last_used_ts = current_time - 2 * 86400 source2 = cache.get_source_by_url("http://delete.example.com") source2.last_used_ts = ( - int(time.time()) - 2 * config_options.DATA_SOURCE_EXPIRATION + current_time - 2 * config_options.DATA_SOURCE_EXPIRATION ) rss2irc.write_cache(cache, fixture_cache_file) # @@ -216,6 +219,10 @@ def test_main_ideal( cache = rss2irc.read_cache(logger, fixture_cache_file) print("Cache: {}".format(cache)) assert list(cache.items.keys()) == expected_cache_keys + for expected_key in expected_cache_keys: + assert expected_key in cache.items + assert cache.items[expected_key]["expiration"] > current_time + assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) assert source.url == rss_url @@ -244,7 +251,10 @@ def test_main_cache_hit( handle = "test" http_timeout = "10" rss_url = "http://rss.example.com" - expected_cache_keys = [] + current_time = int(time.time()) + expected_cache_keys = [ + "https://phpbb.example.com/threads/something.424837/", + ] expected_slack_channel = "test" # Mock/set SLACK_TOKEN @@ -268,7 +278,11 @@ def test_main_cache_hit( source1 = cache.get_source_by_url(rss_url) source1.http_etag = "pytest_etag" source1.http_last_modified = "pytest_lm" - source1.last_used_ts = int(time.time()) - 2 * 86400 + source1.last_used_ts = current_time - 2 * 86400 + cache.items["https://phpbb.example.com/threads/something.424837/"] = { + "expiration": current_time - 2 * 86400, + "comments_cnt": 0, + } rss2irc.write_cache(cache, fixture_cache_file) # authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") @@ -319,6 +333,10 @@ def test_main_cache_hit( cache = rss2irc.read_cache(logger, fixture_cache_file) print("Cache: {}".format(cache)) assert list(cache.items.keys()) == expected_cache_keys + for expected_key in expected_cache_keys: + assert expected_key in cache.items + assert cache.items[expected_key]["expiration"] > current_time + assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) assert source.url == rss_url @@ -334,6 +352,580 @@ def test_main_cache_hit( assert len(fixture_http_server.requests) == 0 +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_slack_token_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that SlackTokenError is handled as expected.""" + expected_log_records = [ + ( + "phpbb2slack", + 40, + "Environment variable SLACK_TOKEN must be set.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + mock_read_cache.return_value = cache + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_cache_read_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that CacheReadError is handled as expected.""" + expected_log_records = [ + ( + "phpbb2slack", + 40, + "Error while reading cache file '/path/not/exist/cache.file'.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + mock_read_cache.side_effect = CacheReadError("pytest") + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2irc.get_rss") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_empty_response_error( + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + monkeypatch, + caplog, +): + """Test that EmptyResponseError is handled as expected.""" + expected_log_records = [ + ( + "phpbb2slack", + 40, + "Got empty response from 'http://rss.example.com'.", + ), + ] + expected_cache_keys = [ + "https://phpbb.example.com/threads/something.424837/", + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + current_time = int(time.time()) + + cache = CachedData() + source1 = cache.get_source_by_url(rss_url) + source1.http_etag = "pytest_etag" + source1.http_last_modified = "pytest_lm" + source1.last_used_ts = current_time - 2 * 86400 + cache.items["https://phpbb.example.com/threads/something.424837/"] = { + "expiration": current_time, + "comments_cnt": 0, + } + + mock_read_cache.return_value = cache + mock_get_rss.side_effect = EmptyResponseError("pytest") + mock_wrap_write_cache.return_value = 0 + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + + mock_get_rss.side_effect = EmptyResponseError("pytest") + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--cache-init", + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-vvv", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + print(caplog.record_tuples) + # Check HTTP RSS mock + mock_get_rss.assert_called_once() + # Check cache and keys in it + print("Cache: {}".format(cache)) + assert list(cache.items.keys()) == expected_cache_keys + for expected_key in expected_cache_keys: + assert expected_key in cache.items + assert cache.items[expected_key]["expiration"] == current_time + + # Check HTTP source + assert rss_url in cache.data_sources.keys() + source = cache.get_source_by_url(rss_url) + assert source.url == rss_url + assert source.http_etag == "pytest_etag" + assert source.http_last_modified == "pytest_lm" + assert source.last_used_ts > int(time.time()) - 60 + assert source.http_error_count == 1 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 0), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_no_news_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + fixture_mock_requests, + monkeypatch, + caplog, +): + """Test that EmptyResponseError is handled as expected.""" + expected_log_records = [] + expected_cache_keys = [ + "https://phpbb.example.com/threads/something.424837/", + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + current_time = int(time.time()) + + cache = CachedData() + source1 = cache.get_source_by_url(rss_url) + source1.http_etag = "pytest_etag" + source1.http_last_modified = "pytest_lm" + source1.last_used_ts = current_time - 2 * 86400 + cache.items["https://phpbb.example.com/threads/something.424837/"] = { + "expiration": current_time - 2 * 86400, + "comments_cnt": 0, + } + + mock_read_cache.return_value = cache + mock_wrap_write_cache.return_value = 0 + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + + rss_fname = os.path.join(SCRIPT_PATH, "files", "rss.xml") + with open(rss_fname, "rb") as fhandle: + rss_data = fhandle.read().decode("utf-8") + + mock_http_rss = fixture_mock_requests.get( + rss_url, + text=rss_data, + headers={ + "ETag": "pytest_etag", + "Last-Modified": "pytest_lm", + }, + ) + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--cache-init", + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + # Check HTTP RSS mock + assert mock_http_rss.called is True + assert mock_http_rss.call_count == 1 + assert mock_http_rss.last_request.text is None + # Check cache and keys in it + print("Cache: {}".format(cache)) + assert list(cache.items.keys()) == expected_cache_keys + for expected_key in expected_cache_keys: + assert expected_key in cache.items + assert cache.items[expected_key]["expiration"] > current_time + + # Check HTTP source + assert rss_url in cache.data_sources.keys() + source = cache.get_source_by_url(rss_url) + assert source.url == rss_url + assert source.http_etag == "pytest_etag" + assert source.http_last_modified == "pytest_lm" + assert source.last_used_ts > int(time.time()) - 60 + assert source.http_error_count == 0 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2slack.get_slack_token") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_random_exception( + mock_read_cache, + mock_get_slack_token, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that unexpected exception is handled correctly.""" + expected_log_records = [ + ( + "phpbb2slack", + 40, + "Unexpected exception has occurred.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + mock_read_cache.return_value = cache + mock_get_slack_token.side_effect = ValueError("pytest") + mock_wrap_write_cache.return_value = 0 + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_slack_token.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + # Check HTTP source errors + source = cache.get_source_by_url(rss_url) + assert source.http_error_count == 1 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("phpbb2slack.rss2irc.wrap_write_cache") +@patch("phpbb2slack.rss2irc.read_cache") +def test_main_wrap_write_cache_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + fixture_mock_requests, + monkeypatch, + caplog, +): + """Test that error in wrap_write_cache is handled as expected.""" + expected_log_records = [] + expected_cache_keys = [ + "https://phpbb.example.com/threads/something-of-something.424837/", + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + mock_read_cache.return_value = cache + mock_wrap_write_cache.return_value = 1 + authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + + rss_fname = os.path.join(SCRIPT_PATH, "files", "phpbb-rss.xml") + with open(rss_fname, "rb") as fhandle: + rss_data = fhandle.read().decode("utf-8") + + mock_http_rss = fixture_mock_requests.get( + rss_url, + text=rss_data, + headers={ + "ETag": "pytest_etag", + "Last-Modified": "pytest_lm", + }, + ) + + exception = None + args = [ + "./phpbb2slack.py", + "--authors-of-interest", + authors_file, + "--cache", + fixture_cache_file, + "--cache-init", + "--handle", + handle, + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + phpbb2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + # Check HTTP RSS mock + assert mock_http_rss.called is True + assert mock_http_rss.call_count == 1 + assert mock_http_rss.last_request.text is None + # Check cache and keys in it + print("Cache: {}".format(cache)) + assert list(cache.items.keys()) == expected_cache_keys + assert rss_url in cache.data_sources.keys() + source = cache.get_source_by_url(rss_url) + assert source.url == rss_url + assert source.http_etag == "pytest_etag" + assert source.http_last_modified == "pytest_lm" + assert source.last_used_ts > int(time.time()) - 60 + # Check HTTP source errors + assert source.http_error_count == 0 + + def test_parse_news(): """Test parse_news().""" expected_news = { diff --git a/tests/test_rss2irc.py b/tests/test_rss2irc.py index d92aeb7..61b1718 100644 --- a/tests/test_rss2irc.py +++ b/tests/test_rss2irc.py @@ -5,13 +5,17 @@ import os import sys import time +from unittest.mock import Mock from unittest.mock import patch import pytest -import rss2irc # noqa: I202 -from lib import CachedData # noqa: I202 -from lib import config_options # noqa: I202 +import rss2irc +from lib import CachedData +from lib import config_options +from lib.exceptions import CacheReadError +from lib.exceptions import CacheWriteError +from lib.exceptions import EmptyResponseError SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -129,6 +133,7 @@ def test_main_ideal( assert list(cache.items.keys()) == expected_cache_keys assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) + assert source.http_error_count == 0 assert source.url == rss_url assert source.http_etag == "pytest_etag" assert source.http_last_modified == "pytest_lm" @@ -235,6 +240,7 @@ def test_main_cache_operations( # Verify data sources assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) + assert source.http_error_count == 0 assert source.url == rss_url assert source.http_etag == "pytest_etag" assert source.http_last_modified == "pytest_lm" @@ -244,6 +250,76 @@ def test_main_cache_operations( assert sorted(output) == sorted(expected_output) +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.read_cache") +@patch("rss2irc.os.path.exists") +def test_main_cache_read_error( + mock_path_exists, + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that CacheReadError is handled as expected.""" + expected_log_records = [ + ( + "rss2irc", + 40, + "Error while reading cache file '/path/not/exist/cache.file'.", + ) + ] + handle = "test" + http_timeout = "10" + rss_url = "http://127.0.0.2:49991" + fixture_cache_file = "/path/not/exist/cache.file" + fixture_output_file = "/path/not/exist/output" + + mock_path_exists.return_value = True + mock_read_cache.side_effect = CacheReadError("pytest") + + args = [ + "./rss2irc.py", + "-v", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--output", + fixture_output_file, + ] + extra_args + + print("URL: {:s}".format(rss_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + print("Output file: {:s}".format(fixture_output_file)) + + exception = None + with patch.object(sys, "argv", args): + try: + rss2irc.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_path_exists.assert_called_with(fixture_output_file) + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + @patch("rss2irc.stat.S_ISFIFO") def test_main_cache_hit( mock_s_isfifo, @@ -251,7 +327,7 @@ def test_main_cache_hit( fixture_cache_file, fixture_output_file, ): - """Test that HTTP Status Code 304 is handled as expected.""" + """Test that NotModified/HTTP Status Code 304 is handled as expected.""" handle = "test" http_timeout = "10" rss_url = "http://rss.example.com" @@ -326,6 +402,7 @@ def test_main_cache_hit( assert list(cache.items.keys()) == expected_cache_keys assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) + assert source.http_error_count == 0 assert source.url == rss_url assert source.http_etag == "pytest_etag" assert source.http_last_modified == "pytest_last_modified" @@ -334,6 +411,381 @@ def test_main_cache_hit( assert sorted(output) == sorted(expected_output) +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +@patch("rss2irc.os.path.exists") +def test_main_empty_response_error( + mock_path_exists, + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that EmptyResponseError is handled as expected.""" + expected_log_records = [ + ( + "rss2irc", + 40, + "Got empty response from 'http://127.0.0.2:49991'.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://127.0.0.2:49991" + fixture_cache_file = "/fake/path/cache.file" + fixture_output_file = "/fake/path/output" + cache_key = "http://example.com" + frozen_ts = int(time.time()) + + cache = CachedData() + cache.items[cache_key] = frozen_ts + 60 + cache.items["https://expired.example.com"] = 123456 + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 0 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + source2 = cache.get_source_by_url("http://delete.example.com") + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION + + mock_path_exists.return_value = True + mock_read_cache.return_value = cache + mock_get_rss.side_effect = EmptyResponseError("pytest") + mock_wrap_write_cache.return_value = 0 + + args = [ + "./rss2irc.py", + "-v", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--output", + fixture_output_file, + ] + extra_args + + print("URL: {:s}".format(rss_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + print("Output file: {:s}".format(fixture_output_file)) + + exception = None + with patch.object(sys, "argv", args): + try: + rss2irc.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_path_exists.assert_called_with(fixture_output_file) + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 1 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 0), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +@patch("rss2irc.os.path.exists") +def test_main_no_news_error( + mock_path_exists, + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that EmptyResponseError is handled as expected.""" + expected_log_records = [ + ( + "rss2irc", + 20, + "No news from 'http://127.0.0.2:49991'?", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://127.0.0.2:49991" + fixture_cache_file = "/fake/path/cache.file" + fixture_output_file = "/fake/path/output" + frozen_ts = int(time.time()) + + cache = CachedData() + cache.items["http://example.com"] = frozen_ts - 3600 + cache.items["https://expired.example.com"] = frozen_ts - 2 * 86400 + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 10 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + + mock_path_exists.return_value = True + mock_read_cache.return_value = cache + mock_rss_fname = os.path.join(SCRIPT_PATH, "files", "rss_no_news.xml") + mock_rsp = Mock() + with open(mock_rss_fname, "r", encoding="utf-8") as fhandle: + mock_rsp.text = fhandle.read() + + mock_get_rss.return_value = mock_rsp + mock_wrap_write_cache.return_value = 0 + + args = [ + "./rss2irc.py", + "-vv", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--output", + fixture_output_file, + ] + extra_args + + print("URL: {:s}".format(rss_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + print("Output file: {:s}".format(fixture_output_file)) + + exception = None + with patch.object(sys, "argv", args): + try: + rss2irc.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_path_exists.assert_called_with(fixture_output_file) + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 10 + # NOTE(zstyblik): check that we have all items and expiration has been + # updated. + assert len(cache.items) == 2 + for key in cache.items: + assert cache.items[key] > frozen_ts + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +@patch("rss2irc.os.path.exists") +def test_main_random_exception( + mock_path_exists, + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that unexpected exception is handled correctly.""" + expected_log_records = [ + ( + "rss2irc", + 40, + "Unexpected exception has occurred.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://127.0.0.2:49991" + fixture_cache_file = "/fake/path/cache.file" + fixture_output_file = "/fake/path/output" + cache_key = "http://example.com" + frozen_ts = int(time.time()) + + cache = CachedData() + cache.items[cache_key] = frozen_ts + 60 + cache.items["https://expired.example.com"] = 123456 + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 0 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + source2 = cache.get_source_by_url("http://delete.example.com") + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION + + mock_path_exists.return_value = True + mock_read_cache.return_value = cache + mock_get_rss.side_effect = ValueError("pytest") + mock_wrap_write_cache.return_value = 0 + + args = [ + "./rss2irc.py", + "-v", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--output", + fixture_output_file, + ] + extra_args + + print("URL: {:s}".format(rss_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + print("Output file: {:s}".format(fixture_output_file)) + + exception = None + with patch.object(sys, "argv", args): + try: + rss2irc.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_path_exists.assert_called_with(fixture_output_file) + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 1 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.stat.S_ISFIFO") +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.read_cache") +def test_main_wrap_write_cache_error( + mock_read_cache, + mock_wrap_write_cache, + mock_s_isfifo, + extra_args, + expected_retcode, + fixture_http_server, + fixture_output_file, +): + """Test that error in wrap_write_cache is handled as expected.""" + expected_cache_keys = [ + "http://www.example.com/scan.php?page=news_item&px=item1", + "http://www.example.com/scan.php?page=news_item&px=item2", + ] + expected_output = [ + ( + b"[test] Item1 | " + b"http://www.example.com/scan.php?page=news_item&px=item1\n" + ), + ( + b"[test] Item2 | " + b"http://www.example.com/scan.php?page=news_item&px=item2\n" + ), + ] + + handle = "test" + http_timeout = "10" + rss_url = fixture_http_server.url + fixture_cache_file = "/fake/path/cache.file" + + cache = CachedData() + mock_read_cache.return_value = cache + mock_wrap_write_cache.return_value = 1 + mock_s_isfifo.return_value = True + + rss_fname = os.path.join(SCRIPT_PATH, "files", "rss.xml") + with open(rss_fname, "rb") as fhandle: + fixture_http_server.serve_content( + fhandle.read().decode("utf-8"), + 200, + {"ETag": "pytest_etag", "Last-Modified": "pytest_lm"}, + ) + + args = [ + "./rss2irc.py", + "-v", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--output", + fixture_output_file, + ] + extra_args + + print("URL: {:s}".format(rss_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + print("Output file: {:s}".format(fixture_output_file)) + + exception = None + with patch.object(sys, "argv", args): + try: + rss2irc.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + + assert list(cache.items.keys()) == expected_cache_keys + assert rss_url in cache.data_sources.keys() + source = cache.get_source_by_url(rss_url) + assert source.http_error_count == 0 + assert source.url == rss_url + assert source.http_etag == "pytest_etag" + assert source.http_last_modified == "pytest_lm" + assert source.last_used_ts > int(time.time()) - 60 + # check output file + with open(fixture_output_file, "rb") as fhandle: + output = fhandle.readlines() + + assert sorted(output) == sorted(expected_output) + + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + + def test_parse_news(): """Test parse_news().""" expected_news = { @@ -392,3 +844,32 @@ def test_update_items_expiration_cache_items_as_news(mock_time): assert cache.items["http://example.com/item1"] == expected_expiration assert cache.items["http://example.com/item2"] == expected_expiration + + +def test_wrap_write_cache(fixture_cache_file): + """Test happy path in wrap_write_cache_exception().""" + cache = CachedData() + logger = logging.getLogger("rss2irc") + + result = rss2irc.wrap_write_cache(logger, cache, fixture_cache_file) + + assert result == 0 + + +@patch("rss2irc.write_cache") +def test_wrap_write_cache_exception(mock_write_cache, caplog): + """Test exception handling in wrap_write_cache_exception().""" + expected_record = ( + "rss2irc", + logging.ERROR, + "Failed to write data into cache file '/path/does/not/exist'.", + ) + cache = CachedData() + cache_file = "/path/does/not/exist" + logger = logging.getLogger("rss2irc") + mock_write_cache.side_effect = CacheWriteError("pytest") + + result = rss2irc.wrap_write_cache(logger, cache, cache_file) + + assert result == 1 + assert expected_record in caplog.record_tuples diff --git a/tests/test_rss2slack.py b/tests/test_rss2slack.py index 870ebff..2968338 100644 --- a/tests/test_rss2slack.py +++ b/tests/test_rss2slack.py @@ -5,14 +5,18 @@ import os import sys import time +from unittest.mock import Mock from unittest.mock import patch import pytest -import rss2irc # noqa: I100, I202 -import rss2slack # noqa: I100, I202 -from lib import CachedData # noqa: I100, I202 -from lib import config_options # noqa: I100, I202 +import rss2irc +import rss2slack +from lib import CachedData +from lib import config_options +from lib.exceptions import CacheReadError +from lib.exceptions import EmptyResponseError +from lib.exceptions import SlackTokenError SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -77,11 +81,11 @@ def test_get_slack_token_no_token(): exception = None try: rss2slack.get_slack_token() - except ValueError as value_error: + except SlackTokenError as value_error: exception = value_error - assert isinstance(exception, ValueError) is True - assert exception.args[0] == "SLACK_TOKEN must be set." + assert isinstance(exception, SlackTokenError) is True + assert exception.args[0] == "SLACK_TOKEN env variable must be set" def test_main_ideal( @@ -234,6 +238,150 @@ def test_main_ideal( assert data == expected_slack_requests[1] +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2slack.rss2irc.wrap_write_cache") +@patch("rss2slack.rss2irc.read_cache") +def test_main_slack_token_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that SlackTokenError is handled as expected.""" + expected_log_records = [ + ( + "rss2slack", + 40, + "Environment variable SLACK_TOKEN must be set.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + mock_read_cache.return_value = CachedData() + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.read_cache") +def test_main_cache_read_error( + mock_read_cache, + mock_wrap_write_cache, + extra_args, + expected_retcode, + monkeypatch, + caplog, +): + """Test that CacheReadError is handled as expected.""" + expected_log_records = [ + ( + "rss2slack", + 40, + "Error while reading cache file '/path/not/exist/cache.file'.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + mock_read_cache.side_effect = CacheReadError("pytest") + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_not_called() + assert caplog.record_tuples == expected_log_records + + def test_main_cache_hit( monkeypatch, fixture_mock_requests, fixture_cache_file, fixture_http_server ): @@ -327,3 +475,373 @@ def test_main_cache_hit( # Check HTTP Slack # Note: this is just a shallow check, but it's better than nothing. assert len(fixture_http_server.requests) == 0 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +def test_main_empty_response_error( + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + monkeypatch, + caplog, +): + """Test that EmptyResponseError is handled as expected.""" + expected_log_records = [ + ( + "rss2slack", + 40, + "Got empty response from 'http://rss.example.com'.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + cache_key = "http://example.com" + frozen_ts = int(time.time()) + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + cache.items[cache_key] = frozen_ts + 60 + cache.items["https://expired.example.com"] = 123456 + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 0 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + source2 = cache.get_source_by_url("http://delete.example.com") + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION + + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + mock_read_cache.return_value = cache + mock_get_rss.side_effect = EmptyResponseError("pytest") + mock_wrap_write_cache.return_value = 0 + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-v", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 1 + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 0), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +def test_main_no_news_error( + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + extra_args, + expected_retcode, + monkeypatch, + caplog, +): + """Test that NoNewsError is handled as expected.""" + expected_log_records = [ + ( + "rss2slack", + 20, + "No news from 'http://rss.example.com'?", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + cache_key = "http://example.com" + frozen_ts = int(time.time()) + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + cache.items[cache_key] = frozen_ts + 60 + cache.items["https://expired.example.com"] = 123456 + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 1 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + source2 = cache.get_source_by_url("http://delete.example.com") + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION + + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + mock_read_cache.return_value = cache + mock_rss_fname = os.path.join(SCRIPT_PATH, "files", "rss_no_news.xml") + mock_rsp = Mock() + with open(mock_rss_fname, "r", encoding="utf-8") as fhandle: + mock_rsp.text = fhandle.read() + + mock_get_rss.return_value = mock_rsp + mock_wrap_write_cache.return_value = 0 + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-vv", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 1 + # NOTE(zstyblik): check that we have all items and expiration has been + # updated. + assert len(cache.items) == 2 + for key in cache.items: + assert cache.items[key] > frozen_ts + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + ([], 0), + (["--return-error"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2slack.get_slack_token") +@patch("rss2irc.read_cache") +def test_main_random_exception( + mock_read_cache, + mock_get_slack_token, + mock_wrap_write_cache, + extra_args, + expected_retcode, + caplog, +): + """Test that unexpected exception is handled as expected.""" + expected_log_records = [ + ( + "rss2slack", + 40, + "Unexpected exception has occurred.", + ), + ] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + cache_key = "http://example.com" + frozen_ts = int(time.time()) + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + cache.items[cache_key] = frozen_ts + cache.items["https://expired.example.com"] = frozen_ts + source1 = cache.get_source_by_url(rss_url) + source1.http_error_count = 1 + source1.http_etag = "" + source1.http_last_modified = "" + source1.last_used_ts = frozen_ts - 2 * 86400 + source2 = cache.get_source_by_url("http://delete.example.com") + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION + + mock_read_cache.return_value = cache + mock_get_slack_token.side_effect = ValueError("pytest") + mock_wrap_write_cache.return_value = 0 + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-vv", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records + assert source1.http_error_count == 2 + # NOTE(zstyblik): check that we have all items and expiration is the same. + assert len(cache.items) == 2 + for key in cache.items: + assert cache.items[key] == frozen_ts + + +@pytest.mark.parametrize( + "extra_args,expected_retcode", + [ + (["--cache-init"], 0), + (["--return-error", "--cache-init"], 1), + ], +) +@patch("rss2irc.wrap_write_cache") +@patch("rss2irc.get_rss") +@patch("rss2irc.read_cache") +def test_main_wrap_write_cache_error( + mock_read_cache, + mock_get_rss, + mock_wrap_write_cache, + monkeypatch, + extra_args, + expected_retcode, + caplog, +): + """Test that error from wrap_write_cache_error() is handled as expected.""" + expected_log_records = [] + handle = "test" + http_timeout = "10" + rss_url = "http://rss.example.com" + slack_base_url = "https://slack.example.com" + expected_slack_channel = "test" + fixture_cache_file = "/path/not/exist/cache.file" + + cache = CachedData() + + # Mock/set SLACK_TOKEN + monkeypatch.setenv("SLACK_TOKEN", "test") + mock_read_cache.return_value = cache + mock_rss_fname = os.path.join(SCRIPT_PATH, "files", "rss.xml") + mock_rsp = Mock() + mock_rsp.headers = {} + with open(mock_rss_fname, "r", encoding="utf-8") as fhandle: + mock_rsp.text = fhandle.read() + + mock_get_rss.return_value = mock_rsp + mock_wrap_write_cache.return_value = 1 + + exception = None + args = [ + "./rss2slack.py", + "--rss-url", + rss_url, + "--rss-http-timeout", + http_timeout, + "--handle", + handle, + "--cache", + fixture_cache_file, + "--slack-base-url", + slack_base_url, + "--slack-channel", + expected_slack_channel, + "--slack-timeout", + "10", + "-vv", + ] + extra_args + + print("RSS URL: {:s}".format(rss_url)) + print("Slack URL: {:s}".format(slack_base_url)) + print("Handle: {:s}".format(handle)) + print("Cache file: {:s}".format(fixture_cache_file)) + + with patch.object(sys, "argv", args): + try: + rss2slack.main() + except SystemExit as sys_exit: + exception = sys_exit + + assert isinstance(exception, SystemExit) is True + assert exception.code == expected_retcode + mock_read_cache.assert_called_once() + mock_get_rss.assert_called_once() + mock_wrap_write_cache.assert_called_once() + assert caplog.record_tuples == expected_log_records