From f7850c6f6b2aafa76131630e4131ef2e1e775468 Mon Sep 17 00:00:00 2001 From: daviz Date: Mon, 22 Jun 2015 20:12:03 +0200 Subject: [PATCH 1/2] first approach to consume foursquare API via search and explore verbs --- search_venues.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++ top_venues.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 search_venues.py create mode 100644 top_venues.py diff --git a/search_venues.py b/search_venues.py new file mode 100644 index 0000000..e7fa56c --- /dev/null +++ b/search_venues.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jun 20 15:23:29 2015 + +@author: daviz +""" + +import foursquare as fq +import json +import csv + +# Construct the client object + +client = fq.Foursquare(client_id='', + client_secret='') + +# Must provide parameters (ll and radius) or (sw and ne) or (near and radius) +response = client.venues.search(params= {'near': "08038, Cataluña, Barcelona",'radius': 1000, + 'intent': 'browse', 'limit': 50, 'offset': 0}) + +# print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) + +with open('dataset/venues_search.csv', 'w') as csvfile: + fieldnames = ['name', 'rating', 'price', 'phone', 'city', 'merchant_zipcode', + 'address', 'latitude', 'longitude', 'category'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for venue in response['venues']: + venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, + 'city': None, 'merchant_zipcode': None, 'address': None, + 'latitude': None, 'longitude': None, 'category': None } + venue_row['name'] = venue['name'] + if 'rating' in venue: + venue_row['rating'] = venue['rating'] + if 'price' in venue: + #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 + venue_row['price'] = venue['price']['tier'] + if 'phone' in venue['contact']: + venue_row['phone']= venue['contact']['phone'] + if 'city' in venue['location']: + venue_row['city'] = venue['location']['city'] + if 'postalCode' in venue['location']: + venue_row['merchant_zipcode'] = venue['location']['postalCode'] + if 'address' in venue['location']: + venue_row['address'] = venue['location']['address'] + if 'lat' in venue['location']: + venue_row['latitude'] = venue['location']['lat'] + if 'lng' in venue['location']: + venue_row['longitude'] = venue['location']['lng'] + for categories in venue['categories']: + category_list = [] + if 'name' in categories: + category_list.append(categories['name']) + venue_row['category'] = ",".join(category_list) + + writer.writerow(venue_row) diff --git a/top_venues.py b/top_venues.py new file mode 100644 index 0000000..a4902b8 --- /dev/null +++ b/top_venues.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jun 20 15:23:29 2015 + +@author: daviz +""" + +import foursquare as fq +import json +import csv + +# Construct the client object + +client = fq.Foursquare(client_id='', + client_secret='') + +#offset = range(0,200,50) +#for +response = client.venues.explore(params= {'ll': "41.359875,2.145081", 'limit': 50, 'offset': 0}) + +# print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) + +with open('dataset/top_venues.csv', 'w') as csvfile: + fieldnames = ['name', 'rating', 'price', 'phone', 'city', 'merchant_zipcode', + 'address', 'latitude', 'longitude', 'category'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for groups in response['groups']: + for group in groups['items']: + venue = group['venue'] + venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, + 'city': None, 'merchant_zipcode': None, 'address': None, + 'latitude': None, 'longitude': None, 'category': None } + venue_row['name'] = venue['name'] + if 'rating' in venue: + venue_row['rating'] = venue['rating'] + if 'price' in venue: + #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 + venue_row['price'] = venue['price']['tier'] + if 'phone' in venue['contact']: + venue_row['phone']= venue['contact']['phone'] + if 'city' in venue['location']: + venue_row['city'] = venue['location']['city'] + if 'postalCode' in venue['location']: + venue_row['merchant_zipcode'] = venue['location']['postalCode'] + if 'address' in venue['location']: + venue_row['address'] = venue['location']['address'] + if 'lat' in venue['location']: + venue_row['latitude'] = venue['location']['lat'] + if 'lng' in venue['location']: + venue_row['longitude'] = venue['location']['lng'] + for categories in venue['categories']: + category_list = [] + if 'name' in categories: + category_list.append(categories['name']) + venue_row['category'] = ",".join(category_list) + + writer.writerow(venue_row) From ac4abd91e57d72f57707034c79e70e44ece2ac4b Mon Sep 17 00:00:00 2001 From: daviz Date: Wed, 24 Jun 2015 21:45:46 +0200 Subject: [PATCH 2/2] imposible to obtain a list of restaurants using zipcodes, or gps coordinates and offset. Always duplicates results between pages --- offset-range-top-venues.py | 61 +++++++++++++++++++++++++++ search_venues.py | 76 ++++++++++++++++++---------------- top_venues.py | 85 +++++++++++++++++++++----------------- 3 files changed, 149 insertions(+), 73 deletions(-) create mode 100644 offset-range-top-venues.py diff --git a/offset-range-top-venues.py b/offset-range-top-venues.py new file mode 100644 index 0000000..beaea9d --- /dev/null +++ b/offset-range-top-venues.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jun 20 15:23:29 2015 + +@author: daviz +""" + +import foursquare as fq +import json +import csv +import os + +# Construct the client object + +client = fq.Foursquare(client_id = os.environ.get('FOURSQUARE_CLIENT_ID'), + client_secret = os.environ.get('FOURSQUARE_CLIENT_SECRET')) + + +# print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) +offset_range= range(0,11,1) +with open('dataset/offset_top_venues.csv', 'w') as csvfile: + fieldnames = ['name', 'rating', 'price', 'phone', 'city', 'merchant_zipcode', + 'address', 'latitude', 'longitude', 'category'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for offset in offset_range: + response = client.venues.explore(params= {'ll': '41.3879, 2.1699', + 'offset': str(offset), + 'limit': '50'}) + for groups in response['groups']: + for group in groups['items']: + venue = group['venue'] + venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, + 'city': None, 'merchant_zipcode': None, 'address': None, + 'latitude': None, 'longitude': None, 'category': None } + venue_row['name'] = venue['name'] + if 'rating' in venue: + venue_row['rating'] = venue['rating'] + if 'price' in venue: + #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 + venue_row['price'] = venue['price']['tier'] + if 'phone' in venue['contact']: + venue_row['phone']= venue['contact']['phone'] + if 'city' in venue['location']: + venue_row['city'] = venue['location']['city'] + if 'postalCode' in venue['location']: + venue_row['merchant_zipcode'] = venue['location']['postalCode'] + if 'address' in venue['location']: + venue_row['address'] = venue['location']['address'] + if 'lat' in venue['location']: + venue_row['latitude'] = venue['location']['lat'] + if 'lng' in venue['location']: + venue_row['longitude'] = venue['location']['lng'] + for categories in venue['categories']: + category_list = [] + if 'name' in categories: + category_list.append(categories['name']) + venue_row['category'] = ",".join(category_list) + + writer.writerow(venue_row) diff --git a/search_venues.py b/search_venues.py index e7fa56c..ad696e4 100644 --- a/search_venues.py +++ b/search_venues.py @@ -8,50 +8,56 @@ import foursquare as fq import json import csv +import os # Construct the client object -client = fq.Foursquare(client_id='', - client_secret='') +client = fq.Foursquare(client_id = os.environ.get('FOURSQUARE_CLIENT_ID'), + client_secret = os.environ.get('FOURSQUARE_CLIENT_SECRET')) + # Must provide parameters (ll and radius) or (sw and ne) or (near and radius) -response = client.venues.search(params= {'near': "08038, Cataluña, Barcelona",'radius': 1000, - 'intent': 'browse', 'limit': 50, 'offset': 0}) -# print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) +# print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) +offset_range= range(0,11,1) with open('dataset/venues_search.csv', 'w') as csvfile: fieldnames = ['name', 'rating', 'price', 'phone', 'city', 'merchant_zipcode', 'address', 'latitude', 'longitude', 'category'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() - - for venue in response['venues']: - venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, - 'city': None, 'merchant_zipcode': None, 'address': None, - 'latitude': None, 'longitude': None, 'category': None } - venue_row['name'] = venue['name'] - if 'rating' in venue: - venue_row['rating'] = venue['rating'] - if 'price' in venue: - #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 - venue_row['price'] = venue['price']['tier'] - if 'phone' in venue['contact']: - venue_row['phone']= venue['contact']['phone'] - if 'city' in venue['location']: - venue_row['city'] = venue['location']['city'] - if 'postalCode' in venue['location']: - venue_row['merchant_zipcode'] = venue['location']['postalCode'] - if 'address' in venue['location']: - venue_row['address'] = venue['location']['address'] - if 'lat' in venue['location']: - venue_row['latitude'] = venue['location']['lat'] - if 'lng' in venue['location']: - venue_row['longitude'] = venue['location']['lng'] - for categories in venue['categories']: - category_list = [] - if 'name' in categories: - category_list.append(categories['name']) - venue_row['category'] = ",".join(category_list) - - writer.writerow(venue_row) + #'ne': "41.443441, 2.184005", 'sw': '41.353175, 2.147098' + for offset in offset_range: + response = client.venues.search(params= {'ll': '41.3879, 2.1699', 'radius': '10000', + 'intent': 'browse', 'llAcc': '10', 'limit': '10', + 'offset': str(offset) }) + + for venue in response['venues']: + venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, + 'city': None, 'merchant_zipcode': None, 'address': None, + 'latitude': None, 'longitude': None, 'category': None } + venue_row['name'] = venue['name'] + if 'rating' in venue: + venue_row['rating'] = venue['rating'] + if 'price' in venue: + #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 + venue_row['price'] = venue['price']['tier'] + if 'phone' in venue['contact']: + venue_row['phone']= venue['contact']['phone'] + if 'city' in venue['location']: + venue_row['city'] = venue['location']['city'] + if 'postalCode' in venue['location']: + venue_row['merchant_zipcode'] = venue['location']['postalCode'] + if 'address' in venue['location']: + venue_row['address'] = venue['location']['address'] + if 'lat' in venue['location']: + venue_row['latitude'] = venue['location']['lat'] + if 'lng' in venue['location']: + venue_row['longitude'] = venue['location']['lng'] + for categories in venue['categories']: + category_list = [] + if 'name' in categories: + category_list.append(categories['name']) + venue_row['category'] = ",".join(category_list) + + writer.writerow(venue_row) diff --git a/top_venues.py b/top_venues.py index a4902b8..202a2ab 100644 --- a/top_venues.py +++ b/top_venues.py @@ -6,17 +6,22 @@ """ import foursquare as fq -import json +#import json import csv +import os +import pandas as pd # Construct the client object +os.environ.get('FOURSQUARE_CLIENT_ID') +client = fq.Foursquare(client_id = os.environ.get('FOURSQUARE_CLIENT_ID'), + client_secret = os.environ.get('FOURSQUARE_CLIENT_SECRET')) -client = fq.Foursquare(client_id='', - client_secret='') - -#offset = range(0,200,50) -#for -response = client.venues.explore(params= {'ll': "41.359875,2.145081", 'limit': 50, 'offset': 0}) +bcn_zipcodes = ['08001', '08002', '08003', '08004', '08005', '08006', '08007', + '08008', '08009', '08010', '08011', '08012', '08013', '08014', + '08015', '08016', '08017', '08018', '08019', '08020', '08021', + '08022', '08023', '08024', '08025', '08026', '08027', '08028', + '08029', '08030', '08031', '08032', '08033', '08034', '08035', + '08036', '08037', '08038', '08039', '08040', '08041', '08042'] # print json.dumps(response['groups'] ,indent=2, separators=(',', ': ')) @@ -25,35 +30,39 @@ 'address', 'latitude', 'longitude', 'category'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() + for zipcode in bcn_zipcodes: + response = client.venues.explore(params= {'near': zipcode + ", Barcelona, Cataluña", 'limit': 50}) + for groups in response['groups']: + for group in groups['items']: + venue = group['venue'] + venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, + 'city': None, 'merchant_zipcode': None, 'address': None, + 'latitude': None, 'longitude': None, 'category': None } + venue_row['name'] = venue['name'] + if 'rating' in venue: + venue_row['rating'] = venue['rating'] + if 'price' in venue: + #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 + venue_row['price'] = venue['price']['tier'] + if 'phone' in venue['contact']: + venue_row['phone']= venue['contact']['phone'] + if 'city' in venue['location']: + venue_row['city'] = venue['location']['city'] + if 'postalCode' in venue['location']: + venue_row['merchant_zipcode'] = venue['location']['postalCode'] + if 'address' in venue['location']: + venue_row['address'] = venue['location']['address'] + if 'lat' in venue['location']: + venue_row['latitude'] = venue['location']['lat'] + if 'lng' in venue['location']: + venue_row['longitude'] = venue['location']['lng'] + for categories in venue['categories']: + category_list = [] + if 'name' in categories: + category_list.append(categories['name']) + venue_row['category'] = ",".join(category_list) - for groups in response['groups']: - for group in groups['items']: - venue = group['venue'] - venue_row = { 'name': None, 'rating': None, 'price': None, 'phone': None, - 'city': None, 'merchant_zipcode': None, 'address': None, - 'latitude': None, 'longitude': None, 'category': None } - venue_row['name'] = venue['name'] - if 'rating' in venue: - venue_row['rating'] = venue['rating'] - if 'price' in venue: - #1 is < $10 an entree, 2 is $10-$20 an entree, 3 is $20-$30 an entree, 4 is > $30 - venue_row['price'] = venue['price']['tier'] - if 'phone' in venue['contact']: - venue_row['phone']= venue['contact']['phone'] - if 'city' in venue['location']: - venue_row['city'] = venue['location']['city'] - if 'postalCode' in venue['location']: - venue_row['merchant_zipcode'] = venue['location']['postalCode'] - if 'address' in venue['location']: - venue_row['address'] = venue['location']['address'] - if 'lat' in venue['location']: - venue_row['latitude'] = venue['location']['lat'] - if 'lng' in venue['location']: - venue_row['longitude'] = venue['location']['lng'] - for categories in venue['categories']: - category_list = [] - if 'name' in categories: - category_list.append(categories['name']) - venue_row['category'] = ",".join(category_list) - - writer.writerow(venue_row) + writer.writerow(venue_row) +top_venues = pd.read_csv("dataset/top_venues.csv") +top_venues.drop_duplicates(inplace=True) +top_venues.to_csv("dataset/top_venues.csv")