diff --git a/play_scraper/api.py b/play_scraper/api.py index e4c1694..bf32752 100644 --- a/play_scraper/api.py +++ b/play_scraper/api.py @@ -11,7 +11,7 @@ from play_scraper import scraper -def details(app_id, hl="en", gl="us"): +def details(app_id, hl="en", gl="us", **kwargs): """Sends a GET request to the app's info page, parses the app's details, and returns them as a dict. @@ -19,7 +19,7 @@ def details(app_id, hl="en", gl="us"): :return: a dictionary of app details """ s = scraper.PlayScraper(hl, gl) - return s.details(app_id) + return s.details(app_id, **kwargs) def collection(collection, category=None, hl="en", gl="us", **kwargs): @@ -55,7 +55,7 @@ def developer(developer, hl="en", gl="us", **kwargs): return s.developer(developer, **kwargs) -def suggestions(query, hl="en", gl="us"): +def suggestions(query, hl="en", gl="us", **kwargs): """Sends a GET request to the Play Store's suggestion API and returns up to five autocompleted suggested query strings in a list. @@ -63,10 +63,10 @@ def suggestions(query, hl="en", gl="us"): :return: a list of suggestion strings """ s = scraper.PlayScraper(hl, gl) - return s.suggestions(query) + return s.suggestions(query, **kwargs) -def search(query, page=None, detailed=False, hl="en", gl="us"): +def search(query, page=None, detailed=False, hl="en", gl="us", **kwargs): """Sends a POST request and retrieves a list of applications matching the query term(s). @@ -76,10 +76,10 @@ def search(query, page=None, detailed=False, hl="en", gl="us"): :return: a list of apps matching search terms """ s = scraper.PlayScraper(hl, gl) - return s.search(query, page, detailed) + return s.search(query, page, detailed, **kwargs) -def similar(app_id, detailed=False, hl="en", gl="us"): +def similar(app_id, detailed=False, hl="en", gl="us", **kwargs): """Sends a GET request, follows the redirect, and retrieves a list of applications similar to the specified app. @@ -88,14 +88,14 @@ def similar(app_id, detailed=False, hl="en", gl="us"): :return: a list of similar apps """ s = scraper.PlayScraper(hl, gl) - return s.similar(app_id, detailed=detailed) + return s.similar(app_id, detailed=detailed, **kwargs) -def categories(hl="en", gl="us", ignore_promotions=True): +def categories(hl="en", gl="us", ignore_promotions=True, **kwargs): """Sends a GET request to the front page (app store base url), parses and returns a list of all available categories. Note: May contain some promotions, e.g. "Popular Characters" """ s = scraper.PlayScraper(hl, gl) - return s.categories(ignore_promotions) + return s.categories(ignore_promotions, **kwargs) diff --git a/play_scraper/scraper.py b/play_scraper/scraper.py index b0d5992..058e582 100644 --- a/play_scraper/scraper.py +++ b/play_scraper/scraper.py @@ -75,7 +75,7 @@ def _parse_multiple_apps(self, list_response): return multi_futures_app_request(app_ids, params=self.params) - def details(self, app_id): + def details(self, app_id, **kwargs): """Sends a GET request and parses an application's details. :param app_id: the app to retrieve details, e.g. 'com.nintendo.zaaa' @@ -83,8 +83,10 @@ def details(self, app_id): """ url = build_url("details", app_id) + kwargs.update({"params": self.params}) + try: - response = send_request("GET", url, params=self.params) + response = send_request("GET", url, **kwargs) soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") except requests.exceptions.HTTPError as e: raise ValueError( @@ -103,6 +105,7 @@ def collection( page=None, age=None, detailed=False, + **kwargs ): """Sends a POST request and fetches a list of applications belonging to the collection and an optional category. @@ -142,7 +145,8 @@ def collection( url = build_collection_url(category, collection_name) data = generate_post_data(results, page) - response = send_request("POST", url, data, self.params) + kwargs.update({"params": self.params}) + response = send_request("POST", url, data, **kwargs) if detailed: apps = self._parse_multiple_apps(response) @@ -155,7 +159,7 @@ def collection( return apps - def developer(self, developer, results=None, page=None, detailed=False): + def developer(self, developer, results=None, page=None, detailed=False, **kwargs): """Sends a POST request and retrieves a list of the developer's published applications on the Play Store. @@ -181,7 +185,8 @@ def developer(self, developer, results=None, page=None, detailed=False): url = build_url("developer", developer) data = generate_post_data(results, 0, pagtok) - response = send_request("POST", url, data, self.params) + kwargs.update({"params": self.params}) + response = send_request("POST", url, data, **kwargs) if detailed: apps = self._parse_multiple_apps(response) @@ -193,7 +198,7 @@ def developer(self, developer, results=None, page=None, detailed=False): return apps - def suggestions(self, query): + def suggestions(self, query, **kwargs): """Sends a GET request and retrieves a list of autocomplete suggestions matching the query term(s). @@ -205,11 +210,12 @@ def suggestions(self, query): self.params.update({"json": 1, "c": 0, "query": query}) - response = send_request("GET", self._suggestion_url, params=self.params) + kwargs.update({"params": self.params}) + response = send_request("GET", self._suggestion_url, **kwargs) suggestions = [q["s"] for q in response.json()] return suggestions - def search(self, query, page=None, detailed=False): + def search(self, query, page=None, detailed=False, **kwargs): """Sends a POST request and retrieves a list of applications matching the query term(s). @@ -228,8 +234,9 @@ def search(self, query, page=None, detailed=False): data = generate_post_data(0, 0, pagtok) self.params.update({"q": quote_plus(query), "c": "apps"}) + kwargs.update({"params": self.params, "data": data}) - response = send_request("POST", self._search_url, data, self.params) + response = send_request("POST", self._search_url, **kwargs) soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") if detailed: @@ -248,7 +255,9 @@ def similar(self, app_id, detailed=False, **kwargs): :return: a list of similar apps """ url = build_url("similar", app_id) - response = send_request("GET", url, params=self.params, allow_redirects=True) + kwargs.update({"params": self.params, "allow_redirects": True}) + + response = send_request("GET", url, **kwargs) soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") if detailed: @@ -258,13 +267,14 @@ def similar(self, app_id, detailed=False, **kwargs): return apps - def categories(self, ignore_promotions=True): + def categories(self, ignore_promotions=True, **kwargs): """Sends a GET request to the front page (app store base url), parses and returns a list of all available categories. """ categories = {} - response = send_request("GET", s.BASE_URL, params=self.params) + kwargs.update({"params": self.params}) + response = send_request("GET", s.BASE_URL, **kwargs) soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") category_links = soup.select( diff --git a/play_scraper/utils.py b/play_scraper/utils.py index 917b65e..ed81ee6 100644 --- a/play_scraper/utils.py +++ b/play_scraper/utils.py @@ -94,6 +94,7 @@ def send_request( timeout=30, verify=True, allow_redirects=False, + **kwargs, ): """Sends a request to the url and returns the response. @@ -121,6 +122,7 @@ def send_request( timeout=timeout, verify=verify, allow_redirects=allow_redirects, + **kwargs, ) if not response.status_code == requests.codes.ok: response.raise_for_status() diff --git a/tox.ini b/tox.ini index 932fb47..8b5d37d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ envlist = py27, py32, py33, py34, py35, py36, py37 [testenv] -deps = -rrequirements.txt +deps = -r requirements.txt usedevelop = True commands = python -m unittest discover -