Add proxy support

play_scraper/api.py

@@ -11,15 +11,15 @@
 from play_scraper import scraper
 
 
-def details(app_id, hl="en", gl="us"):
+def details(app_id, hl="en", gl="us", **kwargs):
     """Sends a GET request to the app's info page, parses the app's details, and
     returns them as a dict.
 
     :param app_id: the app to retrieve details from, e.g. 'com.nintendo.zaaa'
     :return: a dictionary of app details
     """
     s = scraper.PlayScraper(hl, gl)
-    return s.details(app_id)
+    return s.details(app_id, **kwargs)
 
 
 def collection(collection, category=None, hl="en", gl="us", **kwargs):
@@ -55,18 +55,18 @@ def developer(developer, hl="en", gl="us", **kwargs):
     return s.developer(developer, **kwargs)
 
 
-def suggestions(query, hl="en", gl="us"):
+def suggestions(query, hl="en", gl="us", **kwargs):
     """Sends a GET request to the Play Store's suggestion API and returns up to
     five autocompleted suggested query strings in a list.
 
     :param query: the query string to get autocomplete suggestions
     :return: a list of suggestion strings
     """
     s = scraper.PlayScraper(hl, gl)
-    return s.suggestions(query)
+    return s.suggestions(query, **kwargs)
 
 
-def search(query, page=None, detailed=False, hl="en", gl="us"):
+def search(query, page=None, detailed=False, hl="en", gl="us", **kwargs):
     """Sends a POST request and retrieves a list of applications matching
     the query term(s).
 
@@ -76,10 +76,10 @@ def search(query, page=None, detailed=False, hl="en", gl="us"):
     :return: a list of apps matching search terms
     """
     s = scraper.PlayScraper(hl, gl)
-    return s.search(query, page, detailed)
+    return s.search(query, page, detailed, **kwargs)
 
 
-def similar(app_id, detailed=False, hl="en", gl="us"):
+def similar(app_id, detailed=False, hl="en", gl="us", **kwargs):
     """Sends a GET request, follows the redirect, and retrieves a list of
     applications similar to the specified app.
 
@@ -88,14 +88,14 @@ def similar(app_id, detailed=False, hl="en", gl="us"):
     :return: a list of similar apps
     """
     s = scraper.PlayScraper(hl, gl)
-    return s.similar(app_id, detailed=detailed)
+    return s.similar(app_id, detailed=detailed, **kwargs)
 
 
-def categories(hl="en", gl="us", ignore_promotions=True):
+def categories(hl="en", gl="us", ignore_promotions=True, **kwargs):
     """Sends a GET request to the front page (app store base url), parses and
     returns a list of all available categories.
 
     Note: May contain some promotions, e.g. "Popular Characters"
     """
     s = scraper.PlayScraper(hl, gl)
-    return s.categories(ignore_promotions)
+    return s.categories(ignore_promotions, **kwargs)

play_scraper/scraper.py

@@ -75,16 +75,18 @@ def _parse_multiple_apps(self, list_response):
 
         return multi_futures_app_request(app_ids, params=self.params)
 
-    def details(self, app_id):
+    def details(self, app_id, **kwargs):
         """Sends a GET request and parses an application's details.
 
         :param app_id: the app to retrieve details, e.g. 'com.nintendo.zaaa'
         :return: a dictionary of app details
         """
         url = build_url("details", app_id)
 
+        kwargs.update({"params": self.params})
+
         try:
-            response = send_request("GET", url, params=self.params)
+            response = send_request("GET", url, **kwargs)
             soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8")
         except requests.exceptions.HTTPError as e:
             raise ValueError(
@@ -103,6 +105,7 @@ def collection(
         page=None,
         age=None,
         detailed=False,
+        **kwargs
     ):
         """Sends a POST request and fetches a list of applications belonging to
         the collection and an optional category.
@@ -142,7 +145,8 @@ def collection(
 
         url = build_collection_url(category, collection_name)
         data = generate_post_data(results, page)
-        response = send_request("POST", url, data, self.params)
+        kwargs.update({"params": self.params})
+        response = send_request("POST", url, data, **kwargs)
 
         if detailed:
             apps = self._parse_multiple_apps(response)
@@ -155,7 +159,7 @@ def collection(
 
         return apps
 
-    def developer(self, developer, results=None, page=None, detailed=False):
+    def developer(self, developer, results=None, page=None, detailed=False, **kwargs):
         """Sends a POST request and retrieves a list of the developer's
         published applications on the Play Store.
 
@@ -181,7 +185,8 @@ def developer(self, developer, results=None, page=None, detailed=False):
 
         url = build_url("developer", developer)
         data = generate_post_data(results, 0, pagtok)
-        response = send_request("POST", url, data, self.params)
+        kwargs.update({"params": self.params})
+        response = send_request("POST", url, data, **kwargs)
 
         if detailed:
             apps = self._parse_multiple_apps(response)
@@ -193,7 +198,7 @@ def developer(self, developer, results=None, page=None, detailed=False):
 
         return apps
 
-    def suggestions(self, query):
+    def suggestions(self, query, **kwargs):
         """Sends a GET request and retrieves a list of autocomplete suggestions
         matching the query term(s).
 
@@ -205,11 +210,12 @@ def suggestions(self, query):
 
         self.params.update({"json": 1, "c": 0, "query": query})
 
-        response = send_request("GET", self._suggestion_url, params=self.params)
+        kwargs.update({"params": self.params})
+        response = send_request("GET", self._suggestion_url, **kwargs)
         suggestions = [q["s"] for q in response.json()]
         return suggestions
 
-    def search(self, query, page=None, detailed=False):
+    def search(self, query, page=None, detailed=False, **kwargs):
         """Sends a POST request and retrieves a list of applications matching
         the query term(s).
 
@@ -228,8 +234,9 @@ def search(self, query, page=None, detailed=False):
         data = generate_post_data(0, 0, pagtok)
 
         self.params.update({"q": quote_plus(query), "c": "apps"})
+        kwargs.update({"params": self.params, "data": data})
 
-        response = send_request("POST", self._search_url, data, self.params)
+        response = send_request("POST", self._search_url, **kwargs)
         soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8")
 
         if detailed:
@@ -248,7 +255,9 @@ def similar(self, app_id, detailed=False, **kwargs):
         :return: a list of similar apps
         """
         url = build_url("similar", app_id)
-        response = send_request("GET", url, params=self.params, allow_redirects=True)
+        kwargs.update({"params": self.params, "allow_redirects": True})
+
+        response = send_request("GET", url, **kwargs)
         soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8")
 
         if detailed:
@@ -258,13 +267,14 @@ def similar(self, app_id, detailed=False, **kwargs):
 
         return apps
 
-    def categories(self, ignore_promotions=True):
+    def categories(self, ignore_promotions=True, **kwargs):
         """Sends a GET request to the front page (app store base url), parses
         and returns a list of all available categories.
         """
         categories = {}
 
-        response = send_request("GET", s.BASE_URL, params=self.params)
+        kwargs.update({"params": self.params})
+        response = send_request("GET", s.BASE_URL, **kwargs)
         soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8")
 
         category_links = soup.select(

play_scraper/utils.py

@@ -94,6 +94,7 @@ def send_request(
     timeout=30,
     verify=True,
     allow_redirects=False,
+    **kwargs,
 ):
     """Sends a request to the url and returns the response.
 
@@ -121,6 +122,7 @@ def send_request(
             timeout=timeout,
             verify=verify,
             allow_redirects=allow_redirects,
+            **kwargs,
         )
         if not response.status_code == requests.codes.ok:
             response.raise_for_status()

tox.ini

@@ -2,8 +2,7 @@
 envlist = py27, py32, py33, py34, py35, py36, py37
 
 [testenv]
-deps = -rrequirements.txt
+deps = -r requirements.txt
 usedevelop = True
 commands =
     python -m unittest discover
-