From 2562e6df967ed7e638ed068e86fe9eae7275628c Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Thu, 25 Jan 2018 21:07:00 -0600 Subject: [PATCH 1/6] Updated for BGG 2018 --- README.rst | 3 + bggcli/__init__.py | 96 +++++++ bggcli/commands/collection_import.py | 66 ++++- bggcli/ui/__init__.py | 21 +- bggcli/ui/gamepage.py | 415 ++++++++++++++++++++++----- 5 files changed, 519 insertions(+), 82 deletions(-) diff --git a/README.rst b/README.rst index ed139ca..adcb1df 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,10 @@ Introduction Only 3 operations are implemented at this time: +Updated for BGG 2018 * bulk import/update for your game collection from a CSV file + +Not tested with BGG 2018 * bulk delete from a CSV file * bulk export as a CSV file, WITH version information (game's version is missing in the default export) diff --git a/bggcli/__init__.py b/bggcli/__init__.py index eac5cdf..30f8a80 100644 --- a/bggcli/__init__.py +++ b/bggcli/__init__.py @@ -1,3 +1,6 @@ +# Updated BGG_SUPPORTED_FIELDS +# Added BGG_EXPORTED_FIELDS and BGG_GAMEDB_FIELDS + import os if os.environ.get('CI') == 'true': @@ -18,3 +21,96 @@ 'pp_currency', 'currvalue', 'cv_currency', 'acquisitiondate', 'acquiredfrom', 'quantity', 'privatecomment', '_versionid'] + +BGG_SUPPORTED_FIELDS = ['own', + 'want', 'wanttobuy', 'wanttoplay', 'prevowned', + 'preordered', + 'fortrade', 'conditiontext', # these must be in this order + 'wishlist', 'wishlistpriority', 'wishlistcomment', # these must be in this order + 'comment', 'rating', + 'pricepaid', 'currvalue', + 'acquisitiondate', 'acquiredfrom', 'quantity', 'privatecomment', + 'haspartslist', 'wantpartslist','publisherid', 'imageid', + 'year', 'language', 'other', + #'cv_currency', 'pp_currency', + 'objectname', + 'objectid', '_versionid', 'invlocation' + ] + +# More fields in the add/edit collection dialog: +# Inventory Date +# Inventory Location +# Barcode +# Aquisition Date +# Inventory Date +# Inventory Location +# Custom Title +# Custom Image Id +# Publisher Id +# Year +# Other +# Barcode + +# list of headings in direct downloaded collection.csv +BGG_EXPORTED_FIELDS = [ +'objectname', +'objectid', +'rating', +'numplays', +'weight', +'own', +'fortrade', +'want', +'wanttobuy', +'wanttoplay', +'prevowned', +'preordered', +'wishlist', +'wishlistpriority', +'wishlistcomment', +'comment', +'conditiontext', +'haspartslist', +'wantpartslist', +'numowned', +'publisherid', +'imageid', +'year', +'language', +'other', +'pricepaid', +'pp_currency', +'currvalue', +'cv_currency', +'acquisitiondate', +'acquiredfrom', +'quantity', +'privatecomment', +'version_publishers', +'version_languages', +'version_yearpublished', +'version_nickname', +] + +BGG_GAMEDB_FIELDS = [ +'collid', +'baverage', +'average', +'avgweight', +'rank', +'objecttype', +'originalname', +'minplayers', +'maxplayers', +'playingtime', +'maxplaytime', +'minplaytime', +'yearpublished', +'bggrecplayers', +'bggbestplayers', +'bggrecagerange', +'bgglanguagedependence', +'itemtype', +] + +#print(set(BGG_EXPORTED_FIELDS)-set(BGG_SUPPORTED_FIELDS)) diff --git a/bggcli/commands/collection_import.py b/bggcli/commands/collection_import.py index cdaaf65..870c252 100644 --- a/bggcli/commands/collection_import.py +++ b/bggcli/commands/collection_import.py @@ -22,6 +22,7 @@ Arguments: The CSV file with games to import """ +# Updated for BGG 2018 import sys from bggcli.commands import check_file from bggcli.ui.gamepage import GamePage @@ -29,23 +30,80 @@ from bggcli.util.csvreader import CsvReader from bggcli.util.logger import Logger from bggcli.util.webdriver import WebDriver +import traceback - +LOOPLIMIT = 30 def execute(args, options): + print('Executing!') login = args['--login'] file_path = check_file(args) csv_reader = CsvReader(file_path) csv_reader.open() - + rows = [] + try: + csv_reader.iterate(lambda row: rows.append(row)) + except: + pass + + print len(rows) + # interestingrows=rows[130:] + # print len(interestingrows) + #sys.exit() + Logger.info("Importing games for '%s' account..." % login) with WebDriver('collection-import', args, options) as web_driver: if not LoginPage(web_driver.driver).authenticate(login, args['--password']): sys.exit(1) - Logger.info("Importing %s games..." % csv_reader.rowCount) + Logger.info("Importing %s games to collection..." % csv_reader.rowCount) game_page = GamePage(web_driver.driver) - csv_reader.iterate(lambda row: game_page.update(row)) + #csv_reader.iterate(lambda row: game_page.update(row)) + #badrows = [] + #rows = rows[:5] + rows.reverse() + firstrow = rows[0] + loop = 1 + Logger.info('Loop {}'.format(loop)) + if loop > LOOPLIMIT: + Logger.info("Loop limit of {} reached.".format(loop)) + return + while len(rows): + row = rows.pop() + Logger.info('Name: {}'.format(row['objectname'])) + + if firstrow is None or firstrow == row: + loop += 1 + Logger.info('Loop {}'.format(loop)) + if rows: + firstrow = rows[0] + Logger.info('First assigned {}'.format(firstrow['objectname'])) + else: + firstrow = None + Logger.info('First assigned None') + try: + val = game_page.update(row) + Logger.info('update returned {}'.format(val)) + + if val: + Logger.info('Updated!') + else: + Logger.info('returned False??, back in queue.') + rows.insert(0,row) + + except Exception as e: + traceback.print_exc(limit=2, file=sys.stdout) + + Logger.info('Exception occurred, back in queue.') + rows.insert(0,row) + + #badrows.append(row) + # for row in rows: + # try: + # game_page.update(row) + # except: + # badrows.append(row) + # print Logger.info("Import has finished.") diff --git a/bggcli/ui/__init__.py b/bggcli/ui/__init__.py index 027e8ec..aa4056b 100644 --- a/bggcli/ui/__init__.py +++ b/bggcli/ui/__init__.py @@ -1,3 +1,6 @@ +# Modified update_select() by adding by_index parameter +# Modified timeout from 5 to 15 seconds + import os from selenium.webdriver.common.by import By @@ -14,7 +17,7 @@ def __init__(self, driver): if os.environ.get('CI') == 'true': timeout = 60 else: - timeout = 5 + timeout = 15 self.wait = WebDriverWait(driver, timeout) @staticmethod @@ -29,22 +32,24 @@ def update_checkbox(root_el, xpath, value): elem.click() @staticmethod - def update_select(el, value, by_text=False): + def update_select(el, value, by_text=False, by_index=False): select = Select(el) if value == '': select.select_by_index(0) + elif by_index: + select.select_by_index(value) elif by_text: select.select_by_visible_text(value) else: select.select_by_value(value) def update_textarea(self, root_el, fieldname, value): - root_el.find_element_by_xpath(".//td[@class='collection_%smod editfield']" % fieldname) \ - .click() - form = self.wait.until(EC.element_to_be_clickable( - (By.XPATH, ".//form[contains(@id, '%s')]" % fieldname))) - self.update_text(form.find_element_by_xpath(".//textarea"), value) - form.find_element_by_xpath(".//input[contains(@onclick, 'CE_SaveData')]").click() + #root_el.find_element_by_xpath(".//td[@class='collection_%smod editfield']" % fieldname) \ + # .click() + input = self.wait.until(EC.element_to_be_clickable( + (By.XPATH, ".//textarea[contains(@id, '%s')]" % fieldname))) + self.update_text(input, value) + #form.find_element_by_xpath(".//input[contains(@onclick, 'CE_SaveData')]").click() def wait_and_accept_alert(self): self.wait.until(EC.alert_is_present()) diff --git a/bggcli/ui/gamepage.py b/bggcli/ui/gamepage.py index 227ff78..c9a2e94 100644 --- a/bggcli/ui/gamepage.py +++ b/bggcli/ui/gamepage.py @@ -1,3 +1,7 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +# Updated for BGG 2018 """ bgg.gamepage ~~~~~~~~~~~~ @@ -5,21 +9,23 @@ Selenium Page Object to bind the game details page """ - +import time +import inspect from selenium.common.exceptions import NoSuchElementException, WebDriverException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.action_chains import ActionChains from bggcli import BGG_BASE_URL, BGG_SUPPORTED_FIELDS from bggcli.ui import BasePage from bggcli.util.logger import Logger - def in_private_info_popup(func): """ Ensure the Private Info popup is displayed when updating its attributes """ - + # Not verified for BGG 2018 + def _wrapper(self, *args, **kwargs): try: self.itemEl \ @@ -40,7 +46,7 @@ def in_version_popup(func): """ Ensure the Version popup is displayed when updating its attributes """ - + # Not verified for BGG 2018 def _wrapper(self, *args, **kwargs): try: self.itemEl \ @@ -69,6 +75,8 @@ class GamePage(BasePage): # 'year', 'language', 'other', 'pricepaid', 'pp_currency', 'currvalue', 'cv_currency', # 'acquisitiondate', 'acquiredfrom', 'quantity', 'privatecomment', '_versionid' # ] + #objectname objectid rating own fortrade want wanttobuy wanttoplay prevowned preordered wishlist wishlistpriority wishlistcomment comment conditiontext haspartslist wantpartslist publisherid imageid year language other pricepaid pp_currency currvalue cv_currency acquisitiondate acquiredfrom quantity privatecomment _versionid + def __init__(self, driver): BasePage.__init__(self, driver) @@ -77,6 +85,7 @@ def __init__(self, driver): self.privateInfoPopupEl = None self.versionPopupEl = None + # Verified BGG 2018 def goto(self, game_attrs): """ Set Web Driver on the game details page @@ -85,31 +94,221 @@ def goto(self, game_attrs): """ self.driver.get("%s/boardgame/%s" % (BGG_BASE_URL, game_attrs['objectid'])) + # Verified BGG 2018 def update(self, game_attrs): + Logger.info("update()", append=True, break_line=True) + + # First, see if objectid exists. + id = game_attrs.get('objectid',None) + + # if objectid doesn't exist, find it via the name: + # this doesn't work at the moment. All the work is done in updateid() + if id is None: + pass + return self.updateid(game_attrs) + + # Verified BGG 2018 + def updateid(self, game_attrs): """ Update game details :param game_attrs: Game attributes as a dictionary """ + # General use of Selenium is + # A) goto page, + # B) find element using xpath expression, then + # C) take action related to the element. + # + # Flow of this function is: + # 1) Go to game page + # 2) Try to click "Add to Collection" button. + # 3) If it didn't exist, + # a) click the "In Collection" button. + # b) click the "Edit" button + # 4) Open the additional two form dialogs + # (Show Advanced, Show Custom) + # 5) fill out items on form, dependencies show which values must + # exist before the indicated item. + # For example, 'wishlistpriority':{'wishlist':1}, + # means that to set wishlistpriority, then + # wishlist must equal 1. + # 6) Finally, find the form element and submit it. + + + Logger.info("updateid()", append=True, break_line=True) + Logger.info("{} {}, ".format(game_attrs.get('objectid',''),game_attrs.get('objectname','')), append=True, break_line=True) + self.goto(game_attrs) - + + Logger.info("page, ", append=True, break_line=False) + + #
    + # + # In Collection + # self.span = self.driver.find_element_by_xpath( + # "//span[@ng-show='colltoolbarctrl.collection.items.length' and contains(.,'In Collection')]") + + # # self.itemEl = self.driver.find_element_by_xpath( + # # "//table[@class='collectionmodule_table']") + # Logger.info(" (in col'n)", append=True, break_line=False) + try: - self.itemEl = self.driver.find_element_by_xpath( - "//table[@class='collectionmodule_table']") - Logger.info(" (already in collection)", append=True, break_line=False) + Logger.info("edit button? ", append=True, break_line=False) + buttonelement = self.driver.find_element_by_xpath( + "(//button[contains(@ng-click, 'colltoolbarctrl.editItem')])[last()]").click() except NoSuchElementException: - self.driver.find_element_by_xpath( - "(//a[contains(@onclick, 'CE_ModuleAddItem')])[last()]").click() - self.itemEl = self.wait.until(EC.element_to_be_clickable( - (By.XPATH, "//table[@class='collectionmodule_table']"))) - + Logger.info(" not found. ", append=True, break_line=False) + Logger.info("(i.e. game in col'n)...", append=True, break_line=False) + # div = self.driver.find_element_by_xpath( + # "(//div[@class, 'toolbar-actions'])[last()]") + + Logger.info("'In Col'n' button? ", append=True, break_line=False) + button = self.driver.find_element_by_xpath( + '(' + '//' + #'div[@is-open="colltoolbarctrl.isOpen" and descendant::' + #'div[@class="dropdown"]' + #'span[@class="toolbar-action"]' + 'button[@id="button-collection" and descendant::' + 'span[starts-with(@ng-show,"colltoolbarctrl.collection.items.length") and contains(text(),"In Collection")]' + ']' + #']' + ')[last()]' + ) + + Logger.info("Click. ", append=True, break_line=False) + button.click() + + + + #time.sleep(2) + # + + #clickable = self.driver.find_element_by_xpath( + #'(//button[contains(text(),"Edit")])[last()]') + clickable = self.driver.find_element_by_xpath( + '//span[@class="collection-dropdown-item-edit" and //button[contains(text(),"Edit")]]') + Logger.info("Click col'n dropdown. ", append=True, break_line=False) + clickable.click() + #time.sleep(10) + # action = ActionChains(self.driver) + # #action=selenium.interactions.Actions(self.driver); + # #import selenium.webdriver.common.actions.pointer_actions + # #selenium.webdriver.common.actions.pointer_actions.click(clickable) + + + # print (clickable.get_attribute('outerHTML').encode("utf-8")) + # #print (clickable.size) + # time.sleep(2) + # action.move_to_element(clickable) + # action.perform() + # time.sleep(1) + # action.click(clickable) + # action.perform() + # time.sleep(10) +# # clickable.click() + # return + + # //*[@id="mainbody"]/div/div[1]/div[2]/div[2]/ng-include/div/ng-include/div/div/div[2]/div[4]/span[2]/div[1]/span[1]/span/span[2]/span/span/span/button + # $x(“//input[starts-with(@name,’btn’)]”) + # + # self.driver.find_element_by_xpath( + # "(//a[contains(@onclick, 'CE_ModuleAddItem')])[last()]").click() + # self.itemEl = self.wait.until(EC.element_to_be_clickable( + # (By.XPATH, "//table[@class='collectionmodule_table']"))) + + + # /html/body/div[2]/div/div + #
    + + Logger.info("form...", append=True, break_line=False) + self.itemEl = self.wait.until(EC.element_to_be_clickable( + (By.XPATH, "//div[@class='modal-content']"))) + + # Open advanced data entry panel. + # Advanced (private info, parts exchange) + Logger.info("Advanced button...", append=True, break_line=False) + self.wait.until(EC.element_to_be_clickable( + (By.XPATH, + "//a[starts-with(@class,'toggler-caret') and starts-with(@ng-click,'editctrl.showvars.showAdvanced')]" + ))) + try: + Logger.info("finding advanced dropdown...", append=True, break_line=False) + b = self.itemEl.find_element_by_xpath(".//a[@class='toggler-caret' and starts-with(@ng-click,'editctrl.showvars.showAdvanced')]") + Logger.info("Click. ", append=True, break_line=False) + b.click() + except: + Logger.info("Failed.", append=True, break_line=False) + pass + # Open Customize Game Info data entry panel. + # Customize Game Info (title, image) + Logger.info("Custom button...", append=True, break_line=False) + self.wait.until(EC.element_to_be_clickable( + (By.XPATH, + "//a[starts-with(@class,'toggler-caret') and starts-with(@ng-click,'editctrl.showvars.showCustom')]" + ))) + try: + self.itemEl.find_element_by_xpath(".//a[@class='toggler-caret' and starts-with(@ng-click,'editctrl.showvars.showCustom')]").click() + except: + Logger.info("Failed. ", append=True, break_line=False) + + # custom = self.wait.until(EC.element_to_be_clickable( + # (By.XPATH, ".//a[starts-with(@ng-click,'editctrl.showvars.showCustom')]"))) + # custom.click() + + # + Logger.info("Name field...", append=True, break_line=False) + nickname = self.wait.until(EC.element_to_be_clickable( + (By.XPATH, './/input[@ng-model="editctrl.editdata.item.textfield.customname.value"]'))) + + # self.itemEl = self.wait.until(EC.element_to_be_clickable( + # (By.XPATH, "//form[@name='collectioneditorform']"))) # Fill all provided values using dynamic invocations 'fill_[fieldname]' - for key in game_attrs: - if key in BGG_SUPPORTED_FIELDS: - value = game_attrs[key] - if value is not None: - getattr(self, "fill_%s" % key)(value) + dependencies = { + 'wishlistpriority':{'wishlist':1}, + 'wishlistcomment':{'wishlist':1}, + 'conditiontext':{'fortrade':1} + } + +# 'fortrade', 'conditiontext', # these must be in this order +# 'wishlist', 'wishlistpriority', 'wishlistcomment', # these must be in this order + + Logger.info("Updating fields: ", append=True, break_line=False) + try: + for key in BGG_SUPPORTED_FIELDS: + dontdo = 0 + if key in game_attrs: + value = game_attrs[key] + if value is not None: + #print '{}'.format(key), + #time.sleep(2) + dep = dependencies.get(key,None) + if dep: + #print 'depends on: ', + for k,v in dep.iteritems(): + #print '{}={}?'.format(k,v), + if str(game_attrs[k]) != str(v): + dontdo = 1 + if dontdo: + continue + getattr(self, "fill_%s" % key)(value) + except: + Logger.info("\nEXCEPTION.", append=True, break_line=True) + return False + # + #savebutton = self.itemEl.find_element_by_xpath(".//button[@ng-disabled='editctrl.saving']") + Logger.info("Form?", append=True, break_line=False) + form = self.itemEl.find_element_by_xpath(".//form[@name='collectioneditorform']") + # action=selenium.interactions.Actions(driver); + # import selenium.webdriver.common.actions.pointer_actions + # selenium.webdriver.common.actions.pointer_actions().click(savebutton) + Logger.info("submitting, ", append=True, break_line=False) + form.submit() ; + Logger.info("submitted. ", append=True, break_line=False) + time.sleep(0.1) + # action.moveToElement(savebutton).click().perform(); + return True # Save "Private Info" popup if opened try: self.privateInfoPopupEl.find_element_by_xpath(".//input[@type='submit']").click() @@ -127,13 +326,17 @@ def update(self, game_attrs): else: self.wait.until(EC.element_to_be_clickable( (By.XPATH, ".//td[@class='collection_versionmod editfield']"))) + + return True + # Not verified BGG 2018 def delete(self, game_attrs): """ Delete a game in the collection :param game_attrs: Game attributes as a dictionary. Only the id will be used """ + self.goto(game_attrs) try: del_button = self.driver.find_element_by_xpath("//a[contains(@onclick, " @@ -152,65 +355,118 @@ def delete(self, game_attrs): # Functions are named 'fill_{attribute-name}' # ############################################################################################### + + # Verified BGG 2018 def fill_objectid(self, value): + """Ignore objectid field in the CSV.""" pass + # Verified BGG 2018 def fill_objectname(self, value): + """Ignore objectname field in the CSV.""" pass + # Verified BGG 2018 + def hover (self,element): + """Hover over an element.""" + #wd = webdriver_connection.connection + #element = wd.find_element_by_link_text(self.locator) + hov = ActionChains(self.driver).move_to_element(element) + hov.perform() + + # Verified BGG 2018 def fill_rating(self, value): - td = self.driver.find_element_by_xpath("//td[contains(@id, 'CEcell_rating')]") - td.click() - + # hover over a star, then the input box will appear. Then fill the box + # + star = self.itemEl.find_element_by_xpath('(//i[@class="glyphicon fi-star"])[1]') + #print(star.location_once_scrolled_into_view) + star.location_once_scrolled_into_view + self.hover(star) + # + + # self.update_text(self.wait.until( - EC.element_to_be_clickable((By.XPATH, "//input[@style='editrating']"))), value) - td.find_element_by_xpath(".//input[@type='submit']").click() + EC.element_to_be_clickable((By.XPATH, '//input[@ng-model="editctrl.editdata.item.rating"]'))), value) + + # td = self.driver.find_element_by_xpath("//td[contains(@id, 'CEcell_rating')]") + # td.click() + + # self.update_text(self.wait.until( + # EC.element_to_be_clickable((By.XPATH, "//input[@style='editrating']"))), value) + # td.find_element_by_xpath(".//input[@type='submit']").click() def fill_weight(self, value): self.update_select(self.itemEl.find_element_by_xpath(".//select[@name='weight']"), value) def fill_own(self, value): - self.update_checkbox(self.itemEl, ".//ul[@class='collectionstatus']//input[@name='own']", + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), value) def fill_fortrade(self, value): - self.update_checkbox(self.itemEl, - ".//ul[@class='collectionstatus']//input[@name='fortrade']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_want(self, value): - self.update_checkbox(self.itemEl, ".//ul[@class='collectionstatus']//input[@name='want']", + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), value) def fill_wanttobuy(self, value): - self.update_checkbox(self.itemEl, - ".//ul[@class='collectionstatus']//input[@name='wanttobuy']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_wanttoplay(self, value): - self.update_checkbox(self.itemEl, - ".//ul[@class='collectionstatus']//input[@name='wanttoplay']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_prevowned(self, value): - self.update_checkbox(self.itemEl, - ".//ul[@class='collectionstatus']//input[@name='prevowned']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_preordered(self, value): - self.update_checkbox(self.itemEl, - ".//ul[@class='collectionstatus']//input[@name='preordered']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_wishlist(self, value): - self.update_checkbox(self.itemEl, ".//input[@name='wishlist']", value) + varname = inspect.currentframe().f_code.co_name[5:] + self.update_checkbox(self.itemEl, ".//input[@ng-model='item.status.{}']".format(varname), + value) def fill_wishlistpriority(self, value): - self.update_select( - self.itemEl.find_element_by_xpath(".//select[@name='wishlistpriority']"), value) - + #Wishlist checkbox must be checked for this to be visible + # + element = self.wait.until(EC.element_to_be_clickable( + (By.XPATH, '//select[@ng-model="item.wishlistpriority"]'))) + self.update_select(element, int(value)-1, by_index=True) + # Wishlist 1=musthave;2=lovetohave;3=liketohave;4=thinking about it; 5=Don't buy this + + def fill_invlocation(self, value): + # + #self.update_textarea(self.itemEl, 'invlocation', value) + element = self.wait.until(EC.element_to_be_clickable( + (By.ID, 'invlocation'))) + self.update_text(element,value) def fill_wishlistcomment(self, value): + #Wishlist checkbox must be checked for this to be visible + element = self.wait.until(EC.element_to_be_clickable( + (By.ID, 'wishlistcomment'))) self.update_textarea(self.itemEl, 'wishlistcomment', value) def fill_comment(self, value): + # self.update_textarea(self.itemEl, 'comment', value) def fill_conditiontext(self, value): + # "For Trade" must be selected to see this textarea + # + element = self.wait.until(EC.element_to_be_clickable( + (By.ID, 'conditiontext'))) self.update_textarea(self.itemEl, 'conditiontext', value) def fill_haspartslist(self, value): @@ -219,64 +475,83 @@ def fill_haspartslist(self, value): def fill_wantpartslist(self, value): self.update_textarea(self.itemEl, 'wantpartslist', value) - @in_version_popup + #@in_version_popup def fill__versionid(self, value): if value: - radio_version = self.versionPopupEl.find_element_by_xpath( + radio_version = self.itemEl.find_element_by_xpath( "(.//ul)[1]//input[@value='%s']" % value) radio_version.click() - @in_version_popup + #@in_version_popup def fill_publisherid(self, value): - self.update_text(self.versionPopupEl.find_element_by_name('publisherid'), value) + self.update_text(self.itemEl.find_element_by_id('publisherid'), value) - @in_version_popup + #@in_version_popup def fill_imageid(self, value): - self.update_text(self.versionPopupEl.find_element_by_name('imageid'), value) - - @in_version_popup + # + self.update_text(self.itemEl.find_element_by_id('customimage'), value) + #@in_version_popup def fill_year(self, value): - self.update_text(self.versionPopupEl.find_element_by_name('year'), value) + self.update_text(self.itemEl.find_element_by_id('year'), value) - @in_version_popup + #@in_version_popup def fill_language(self, value): - self.update_select(self.versionPopupEl.find_element_by_name('languageid'), value, + # + + self.update_select(self.itemEl.find_element_by_xpath('//select[@ng-model="editctrl.editdata.item.languageid"]'), value, by_text=True) - @in_version_popup + #@in_version_popup def fill_other(self, value): - self.update_text(self.versionPopupEl.find_element_by_name('other'), value) + self.update_text(self.itemEl.find_element_by_id('other'), value) - @in_private_info_popup + #@in_private_info_popup def fill_pricepaid(self, value): - self.update_text(self.privateInfoPopupEl.find_element_by_name('pricepaid'), value) + # + self.update_text(self.itemEl.find_element_by_id('pricepaid'), value) - @in_private_info_popup + #@in_private_info_popup def fill_pp_currency(self, value): - self.update_select(self.privateInfoPopupEl.find_element_by_name('pp_currency'), value) + # self.update_select(self.itemEl.find_element_by_id('pp_currency'), value) + if value == '': + return + #button = self.itemEl.find_element_by_xpath('//button[@id="pp_currency"]') + self.itemEl.find_element_by_xpath( + '//a[starts-with(@ng-click="editctrl.editdata.item.pp_currency") and contains(text(), "{}")]'.format(value)).click() - @in_private_info_popup + + #@in_private_info_popup def fill_currvalue(self, value): - self.update_text(self.privateInfoPopupEl.find_element_by_name('currvalue'), value) + self.update_text(self.itemEl.find_element_by_id('currvalue'), value) - @in_private_info_popup + #@in_private_info_popup def fill_cv_currency(self, value): - self.update_select(self.privateInfoPopupEl.find_element_by_name('cv_currency'), value) - - @in_private_info_popup + # USD + # USD + if value == '': + return + div = self.itemEl.find_element_by_xpath( + '//div[@class="form-group"]/label[@for="cv_currency"]') + button = self.itemEl.find_element_by_xpath('//button[@id="cv_currency"]').click() + a = self.itemEl.find_element_by_xpath( + '//a[starts-with(@ng-click,"editctrl.editdata.item.cv_currency") and contains(text(), "{}")]'.format(value)) + print (a) + a.click() + + #@in_private_info_popup def fill_acquisitiondate(self, value): self.update_text( - self.privateInfoPopupEl.find_element_by_xpath( - "//input[contains(@id, 'acquisitiondateinput')]"), value) + self.itemEl.find_element_by_xpath( + "//input[contains(@id, 'acquisitiondate')]"), value) - @in_private_info_popup + #@in_private_info_popup def fill_acquiredfrom(self, value): - self.update_text(self.privateInfoPopupEl.find_element_by_name('acquiredfrom'), value) + self.update_text(self.itemEl.find_element_by_id('acquiredfrom'), value) - @in_private_info_popup + #@in_private_info_popup def fill_quantity(self, value): - self.update_text(self.privateInfoPopupEl.find_element_by_name('quantity'), value) + self.update_text(self.itemEl.find_element_by_id('quantity'), value) - @in_private_info_popup + #@in_private_info_popup def fill_privatecomment(self, value): - self.update_text(self.privateInfoPopupEl.find_element_by_name('privatecomment'), value) + self.update_text(self.itemEl.find_element_by_id('privatecomment'), value) From 0c45ecac5d4f3485631d758bd777bf940e1da5d5 Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Sat, 10 Feb 2018 10:32:07 -0600 Subject: [PATCH 2/6] Uses Python 3.6, retries when failed upload. --- bggcli/commands/collection_import.py | 14 +-- bggcli/main.py | 31 ++++- bggcli/ui/gamepage.py | 167 +++++++++++---------------- bggcli/ui/loginpage.py | 24 +++- bggcli/util/csvreader.py | 9 +- tests/commons.py | 2 +- 6 files changed, 131 insertions(+), 116 deletions(-) diff --git a/bggcli/commands/collection_import.py b/bggcli/commands/collection_import.py index 870c252..022a5d2 100644 --- a/bggcli/commands/collection_import.py +++ b/bggcli/commands/collection_import.py @@ -42,12 +42,12 @@ def execute(args, options): csv_reader = CsvReader(file_path) csv_reader.open() rows = [] - try: - csv_reader.iterate(lambda row: rows.append(row)) - except: - pass +# try: + csv_reader.iterate(lambda row: rows.append(row)) +# except: +# pass - print len(rows) + #print(len(rows)) # interestingrows=rows[130:] # print len(interestingrows) #sys.exit() @@ -72,7 +72,7 @@ def execute(args, options): return while len(rows): row = rows.pop() - Logger.info('Name: {}'.format(row['objectname'])) + Logger.info('(BGGID {}) Name: {}'.format(row['objectid'],row['objectname'])) if firstrow is None or firstrow == row: loop += 1 @@ -88,7 +88,7 @@ def execute(args, options): Logger.info('update returned {}'.format(val)) if val: - Logger.info('Updated!') + Logger.info('Updated (BGGID {0}) "{1}"'.format(row['objectid'],row['objectname'])) else: Logger.info('returned False??, back in queue.') rows.insert(0,row) diff --git a/bggcli/main.py b/bggcli/main.py index d11823c..79e1203 100644 --- a/bggcli/main.py +++ b/bggcli/main.py @@ -29,18 +29,28 @@ """ import sys import time - +import importlib from selenium.common.exceptions import WebDriverException from docopt import docopt +from docopt import DocoptExit from bggcli import UI_ERROR_MSG from bggcli.util.logger import Logger from bggcli.version import VERSION - +#import traceback def import_command_module(name): - return __import__('bggcli.commands.%s' % name.replace('-', '_'), fromlist=['bggcli.commands']) + #print(sys.path) + #return __import__('bggcli.commands.%s' % name.replace('-', '_'), fromlist=['bggcli.commands']) + mname = 'bggcli.commands.%s' % name.replace('-', '_') + #print(mname) + #print('val:') + #print('__name__',__name__) + val = importlib.import_module(mname)#,package='bggcli.commands')#, fromlist=['commands']) + #print('val=',val) + return val + def exit_unknown_command(command): @@ -88,13 +98,21 @@ def _main(argv): if command == 'help': show_help(args['']) else: + #print('argv=',argv) execute_command(command, argv) def parse_commad_args(command_module, argv): + #print('dir',dir(command_module)) + #print(command_module.__doc__,argv,VERSION) + #try: result = docopt(command_module.__doc__, argv, version='bggcli %s' % VERSION, options_first=False) - + # except DocoptExit: + # traceback.print_exc() + # #print('hi!') + # raise + #print('result=',result) try: return result, explode_dyn_args(result['-c']) except StandardError: @@ -116,17 +134,22 @@ def execute_command(command, argv): try: command_module = import_command_module(command) + print('command_module',command_module) command_args, command_args_options = parse_commad_args(command_module, argv) if command_args: + print(command_args, command_args_options) command_module.execute(command_args, command_args_options) show_duration(timer_start) except ImportError: + #traceback.print_exc() exit_unknown_command(command) except WebDriverException as e: Logger.error(UI_ERROR_MSG, e) except Exception as e: Logger.error("Encountered an unexpected error, please report the issue to the author", e) + # except: + # traceback.print_exc() if __name__ == '__main__': diff --git a/bggcli/ui/gamepage.py b/bggcli/ui/gamepage.py index c9a2e94..d00ba73 100644 --- a/bggcli/ui/gamepage.py +++ b/bggcli/ui/gamepage.py @@ -19,7 +19,7 @@ from bggcli import BGG_BASE_URL, BGG_SUPPORTED_FIELDS from bggcli.ui import BasePage from bggcli.util.logger import Logger - +import traceback def in_private_info_popup(func): """ Ensure the Private Info popup is displayed when updating its attributes @@ -96,7 +96,7 @@ def goto(self, game_attrs): # Verified BGG 2018 def update(self, game_attrs): - Logger.info("update()", append=True, break_line=True) + #Logger.info("update()", append=True, break_line=True) # First, see if objectid exists. id = game_attrs.get('objectid',None) @@ -106,7 +106,50 @@ def update(self, game_attrs): if id is None: pass return self.updateid(game_attrs) + + def notincollection(self): + """Either returns button or False, suitable for if statement""" + try: + Logger.info("edit button? ", append=True, break_line=False) + button1 = self.driver.find_element_by_xpath( + "(//button[contains(@ng-click, 'colltoolbarctrl.editItem')])[last()]") + return button1 + except NoSuchElementException: + return False + + def openeditform(self): + button = self.notincollection() + if button: + button.click() + else: + Logger.info(" not found. ", append=True, break_line=False) + Logger.info("(i.e. game in col'n)...", append=True, break_line=False) + # div = self.driver.find_element_by_xpath( + # "(//div[@class, 'toolbar-actions'])[last()]") + + Logger.info("'In Col'n' button? ", append=True, break_line=False) + button = self.driver.find_element_by_xpath( + '(' + '//' + 'button[@id="button-collection" and descendant::' + 'span[starts-with(@ng-show,"colltoolbarctrl.collection.items.length") and ' + 'contains(text(),"In Collection")]' + ']' + ')[last()]' + ) + Logger.info("Click. ", append=True, break_line=False) + button.click() + clickable = self.driver.find_element_by_xpath( + '//span[@class="collection-dropdown-item-edit" and //button[contains(text(),"Edit")]]') + Logger.info("Click col'n dropdown. ", append=True, break_line=False) + clickable.click() + + Logger.info("form...", append=True, break_line=False) + + self.itemEl = self.wait.until(EC.element_to_be_clickable( + (By.XPATH, "//div[@class='modal-content']"))) + # Verified BGG 2018 def updateid(self, game_attrs): """ @@ -135,96 +178,15 @@ def updateid(self, game_attrs): # 6) Finally, find the form element and submit it. - Logger.info("updateid()", append=True, break_line=True) - Logger.info("{} {}, ".format(game_attrs.get('objectid',''),game_attrs.get('objectname','')), append=True, break_line=True) + #Logger.info("updateid()", append=True, break_line=True) + #Logger.info("{} {}, ".format(game_attrs.get('objectid',''),game_attrs.get('objectname','')), append=True, break_line=True) self.goto(game_attrs) - Logger.info("page, ", append=True, break_line=False) - - #
      - # - # In Collection - # self.span = self.driver.find_element_by_xpath( - # "//span[@ng-show='colltoolbarctrl.collection.items.length' and contains(.,'In Collection')]") - - # # self.itemEl = self.driver.find_element_by_xpath( - # # "//table[@class='collectionmodule_table']") - # Logger.info(" (in col'n)", append=True, break_line=False) - - try: - Logger.info("edit button? ", append=True, break_line=False) - buttonelement = self.driver.find_element_by_xpath( - "(//button[contains(@ng-click, 'colltoolbarctrl.editItem')])[last()]").click() - except NoSuchElementException: - Logger.info(" not found. ", append=True, break_line=False) - Logger.info("(i.e. game in col'n)...", append=True, break_line=False) - # div = self.driver.find_element_by_xpath( - # "(//div[@class, 'toolbar-actions'])[last()]") - - Logger.info("'In Col'n' button? ", append=True, break_line=False) - button = self.driver.find_element_by_xpath( - '(' - '//' - #'div[@is-open="colltoolbarctrl.isOpen" and descendant::' - #'div[@class="dropdown"]' - #'span[@class="toolbar-action"]' - 'button[@id="button-collection" and descendant::' - 'span[starts-with(@ng-show,"colltoolbarctrl.collection.items.length") and contains(text(),"In Collection")]' - ']' - #']' - ')[last()]' - ) - - Logger.info("Click. ", append=True, break_line=False) - button.click() - - - - #time.sleep(2) - # - - #clickable = self.driver.find_element_by_xpath( - #'(//button[contains(text(),"Edit")])[last()]') - clickable = self.driver.find_element_by_xpath( - '//span[@class="collection-dropdown-item-edit" and //button[contains(text(),"Edit")]]') - Logger.info("Click col'n dropdown. ", append=True, break_line=False) - clickable.click() - #time.sleep(10) - # action = ActionChains(self.driver) - # #action=selenium.interactions.Actions(self.driver); - # #import selenium.webdriver.common.actions.pointer_actions - # #selenium.webdriver.common.actions.pointer_actions.click(clickable) - - - # print (clickable.get_attribute('outerHTML').encode("utf-8")) - # #print (clickable.size) - # time.sleep(2) - # action.move_to_element(clickable) - # action.perform() - # time.sleep(1) - # action.click(clickable) - # action.perform() - # time.sleep(10) -# # clickable.click() - # return - - # //*[@id="mainbody"]/div/div[1]/div[2]/div[2]/ng-include/div/ng-include/div/div/div[2]/div[4]/span[2]/div[1]/span[1]/span/span[2]/span/span/span/button - # $x(“//input[starts-with(@name,’btn’)]”) - # - # self.driver.find_element_by_xpath( - # "(//a[contains(@onclick, 'CE_ModuleAddItem')])[last()]").click() - # self.itemEl = self.wait.until(EC.element_to_be_clickable( - # (By.XPATH, "//table[@class='collectionmodule_table']"))) - - - # /html/body/div[2]/div/div - #
      + Logger.info("page, ", append=False, break_line=False) + + self.openeditform() - Logger.info("form...", append=True, break_line=False) - self.itemEl = self.wait.until(EC.element_to_be_clickable( - (By.XPATH, "//div[@class='modal-content']"))) - # Open advanced data entry panel. # Advanced (private info, parts exchange) Logger.info("Advanced button...", append=True, break_line=False) @@ -286,7 +248,7 @@ def updateid(self, game_attrs): dep = dependencies.get(key,None) if dep: #print 'depends on: ', - for k,v in dep.iteritems(): + for k,v in dep.items(): # for python 2: iteritems() #print '{}={}?'.format(k,v), if str(game_attrs[k]) != str(v): dontdo = 1 @@ -295,15 +257,16 @@ def updateid(self, game_attrs): getattr(self, "fill_%s" % key)(value) except: Logger.info("\nEXCEPTION.", append=True, break_line=True) + traceback.print_exc() return False # #savebutton = self.itemEl.find_element_by_xpath(".//button[@ng-disabled='editctrl.saving']") - Logger.info("Form?", append=True, break_line=False) + Logger.info("Form? ", append=True, break_line=False) form = self.itemEl.find_element_by_xpath(".//form[@name='collectioneditorform']") # action=selenium.interactions.Actions(driver); # import selenium.webdriver.common.actions.pointer_actions # selenium.webdriver.common.actions.pointer_actions().click(savebutton) - Logger.info("submitting, ", append=True, break_line=False) + #Logger.info("submitting, ", append=True, break_line=False) form.submit() ; Logger.info("submitted. ", append=True, break_line=False) time.sleep(0.1) @@ -338,17 +301,27 @@ def delete(self, game_attrs): """ self.goto(game_attrs) + # try: - del_button = self.driver.find_element_by_xpath("//a[contains(@onclick, " - "'CE_ModuleDeleteItem')]") + if self.notincollection(): + Logger.info(" (not in collection)", append=True, break_line=False) + return # Not in collection + self.openeditform() + + more = self.driver.find_element_by_xpath("//button[@uib-tooltip='More options']'") + more.click() + + # + del_button = self.driver.find_element_by_xpath('//button[ng-click="editctrl.deleteItem(editctrl.editdata.item)"]') + del_button.click() except NoSuchElementException: - Logger.info(" (not in collection)", append=True, break_line=False) + Logger.info(" Failed: can't find delete button!", append=True, break_line=False) return - del_button.click() - # Confirm alert message - self.wait_and_accept_alert() + # + # wait_and_accept_alert is a browser thing, not an "on page popup". + #self.wait_and_accept_alert() ############################################################################################### # All following functions are invoked dynamically, for each attribute that can be updated # diff --git a/bggcli/ui/loginpage.py b/bggcli/ui/loginpage.py index 3acdef6..73aea55 100644 --- a/bggcli/ui/loginpage.py +++ b/bggcli/ui/loginpage.py @@ -5,7 +5,11 @@ Selenium Page Object to bind the login page and perform authentication """ -import urllib2 +#import urllib2 +try: + from urllib.parse import quote +except: + from urllib2 import quote from selenium.common.exceptions import NoSuchElementException @@ -48,7 +52,21 @@ def authenticate(self, login, password): def is_authenticated(self, login): try: self.driver.find_element_by_xpath("//div[@class='menu_login']//a[@href='/user/%s']" - % urllib2.quote(login)) + % quote(login)) return True except NoSuchElementException: - return False + # try: # BGG 2018 style when on a boardgame page. + # # + # self.driver.find_element_by_xpath("//button[@login-required]") + # return False + # except NoSuchElementException: + # return True + try: # BGG 2018, when on a boardgame page. + # + self.driver.find_element_by_xpath("//span[@class='hidden-md hidden-lg' and contains(text(),'{}')]".format(quote(login))) + return True + except NoSuchElementException: + return False + +# 'span[starts-with(@ng-show,"colltoolbarctrl.collection.items.length") and contains(text(),"In Collection")]' + diff --git a/bggcli/util/csvreader.py b/bggcli/util/csvreader.py index 99a534f..e5f0375 100644 --- a/bggcli/util/csvreader.py +++ b/bggcli/util/csvreader.py @@ -12,13 +12,14 @@ from bggcli import UI_ERROR_MSG, BGG_SUPPORTED_FIELDS from bggcli.util.logger import Logger - +import codecs class CsvReader: reader = None def __init__(self, file_path): - self.file = open(file_path, 'rU') + #self.file = open(file_path, 'rU') # python 2 + self.file = codecs.open(file_path, mode='rb', encoding='utf-8', errors='replace')#, buffering=1) self.rowCount = 0 @staticmethod @@ -47,8 +48,8 @@ def iterate(self, callback): # Encode in UTF-8 for key in row: value = row[key] - if value is not None: - row[key] = unicode(value, 'utf-8') + # if value is not None: + # row[key] = unicode(value, 'utf-8') # python 2 objectname = row['objectname'] if objectname is None or objectname == "": diff --git a/tests/commons.py b/tests/commons.py index 9075906..efc1bbb 100644 --- a/tests/commons.py +++ b/tests/commons.py @@ -8,4 +8,4 @@ def debug_test(msg=""): - print "[bggcli-test] %s" % msg + print("[bggcli-test] %s" % msg) From 3568e91bda2d39e8955db6873694a882fd0e44f3 Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Sun, 11 Feb 2018 16:35:00 -0600 Subject: [PATCH 3/6] Created from "pipenv lock -r > requirements.txt" --- requirements.txt | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..60a7ca8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +docopt==0.6.2 --hash=sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491 +selenium==3.8.1 --hash=sha256:5acb9cdbc2d1a7fbb3e16a8ce9246211cc371f0367ad9c6bc2273cca60a6b045 --hash=sha256:9abd2dbd4a5e9b778483ce7e5adf1ea9364fcbc29da488e979213c825a1515d3 diff --git a/setup.py b/setup.py index 90bacea..bb77cc9 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,6 @@ def run_tests(self): py_modules=['bggcli'], tests_require=["pytest"], test_suite='test', - url='http://github.org/syllant/bggcli', + url='http://github.org/HiGregSmith/bggcli', version=VERSION ) From b03efd2bd22d4992db87c8a63bb94b4f88ae9d7b Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Wed, 14 Feb 2018 13:13:10 -0600 Subject: [PATCH 4/6] Fixed retry loops and firefox closing (added reopen firefox functionality). Updated documentation and setup. --- README.rst | 19 ++-- bggcli/commands/collection_import.py | 124 ++++++++++++++------------- bggcli/main.py | 4 +- bggcli/version.py | 2 +- setup.py | 6 +- 5 files changed, 76 insertions(+), 79 deletions(-) diff --git a/README.rst b/README.rst index adcb1df..63d47c3 100644 --- a/README.rst +++ b/README.rst @@ -2,13 +2,6 @@ bggcli - Command Line Interface for BoardGameGeek.com ===================================================== -.. image:: https://travis-ci.org/syllant/bggcli.svg?branch=master - :target: https://travis-ci.org/syllant/bggcli - - -.. image:: https://coveralls.io/repos/syllant/bggcli/badge.svg?branch=master - :target: https://coveralls.io/r/syllant/bggcli?branch=master - Introduction ============ @@ -16,6 +9,7 @@ Introduction ``bggcli`` is a Command Line Interface providing automation for tedious tasks on `BoardGameGeek `__ (aka BGG). It relies on the Web UI and not on the `official API `__ which doesn't offer all features available. +Originally written by Sylvain Francois and updated for Python 3.6 and the 2018 BGG interface by Greg Smith. Only 3 operations are implemented at this time: @@ -39,11 +33,8 @@ Warning: Installation ============ -Python 2.7 is required. - -:: +Python 3.6 is required. - pip install bggcli Usage ===== @@ -129,7 +120,8 @@ Ideas for future versions Here are some ideas of additional tasks that could be implemented: * Generic import for collections, based on game names and not on the BGG internal identifier. A confirmation would be - required for each ambiguous name to choose among matching games provided by BGG + required for each ambiguous name to choose among matching games provided by BGG, this is done by BGGUPLOAD + available on github. * Update/Delete for plays * Update/Delete for forum subscriptions @@ -139,8 +131,9 @@ Links * *BoardGameGeek*: http://www.boardgamegeek.com * *Officiel XML API 2*: https://www.boardgamegeek.com/wiki/page/BGG_XML_API2 * *boardgamegeek - A Python API for boardgamegeek.com*: https://github.com/lcosmin/boardgamegeek +* *bggupload - This program helps you interactively find the BGGID from a partial name or inexact match.*: https://github.com/HiGregSmith/bggupload -Final note +Final note (by Sylvain Francois) ========== Does it really deserve such a development? Probably not, but my second goal was to discover the Python ecosystem! \ No newline at end of file diff --git a/bggcli/commands/collection_import.py b/bggcli/commands/collection_import.py index 022a5d2..b415940 100644 --- a/bggcli/commands/collection_import.py +++ b/bggcli/commands/collection_import.py @@ -30,9 +30,11 @@ from bggcli.util.csvreader import CsvReader from bggcli.util.logger import Logger from bggcli.util.webdriver import WebDriver +from selenium.common.exceptions import WebDriverException + import traceback -LOOPLIMIT = 30 +LOOPLIMIT = 5 def execute(args, options): print('Executing!') login = args['--login'] @@ -43,67 +45,69 @@ def execute(args, options): csv_reader.open() rows = [] # try: + Logger.info("Parsing input file '{}'...".format(file_path)) csv_reader.iterate(lambda row: rows.append(row)) -# except: -# pass - - #print(len(rows)) - # interestingrows=rows[130:] - # print len(interestingrows) - #sys.exit() + #Logger.info("Found %s games to put in collection..." % csv_reader.rowCount) + rows.reverse() + firstrow = rows[0] + loop = 0 - Logger.info("Importing games for '%s' account..." % login) - - with WebDriver('collection-import', args, options) as web_driver: - if not LoginPage(web_driver.driver).authenticate(login, args['--password']): - sys.exit(1) - - Logger.info("Importing %s games to collection..." % csv_reader.rowCount) - game_page = GamePage(web_driver.driver) - #csv_reader.iterate(lambda row: game_page.update(row)) - #badrows = [] - #rows = rows[:5] - rows.reverse() - firstrow = rows[0] - loop = 1 - Logger.info('Loop {}'.format(loop)) - if loop > LOOPLIMIT: - Logger.info("Loop limit of {} reached.".format(loop)) - return - while len(rows): - row = rows.pop() - Logger.info('(BGGID {}) Name: {}'.format(row['objectid'],row['objectname'])) - - if firstrow is None or firstrow == row: - loop += 1 - Logger.info('Loop {}'.format(loop)) - if rows: - firstrow = rows[0] - Logger.info('First assigned {}'.format(firstrow['objectname'])) - else: - firstrow = None - Logger.info('First assigned None') - try: - val = game_page.update(row) - Logger.info('update returned {}'.format(val)) - - if val: - Logger.info('Updated (BGGID {0}) "{1}"'.format(row['objectid'],row['objectname'])) - else: - Logger.info('returned False??, back in queue.') - rows.insert(0,row) + Logger.info("Importing {} games to collection of '{}' ...".format(csv_reader.rowCount,login)) + while rows: + try: + with WebDriver('collection-import', args, options) as web_driver: + if not LoginPage(web_driver.driver).authenticate(login, args['--password']): + sys.exit(1) + #input("Kill Firefox, then Press Enter to continue...") + game_page = GamePage(web_driver.driver) + while rows: + row = rows.pop() + if firstrow is None or firstrow == row: + loop += 1 + if loop >= LOOPLIMIT: + Logger.info("Loop limit of {} reached.".format(loop)) + return + Logger.info('Loop {} (maximum {})'.format(loop,LOOPLIMIT)) + if rows: + firstrow = rows[0] + Logger.info('First assigned {}'.format(firstrow['objectname'])) + else: + firstrow = None + Logger.info('First assigned None') + + Logger.info('(BGGID {}) Name: {} ({} game left)'.format(row['objectid'],row['objectname'],len(rows)+1)) + + try: + val = game_page.update(row) + Logger.info('update returned {}'.format(val)) + + if val: + #Logger.info('Updated (BGGID {0}) "{1}"'.format(row['objectid'],row['objectname'])) + Logger.info('(BGGID {}) Name: {} UPDATED!'.format(row['objectid'],row['objectname'],len(rows))) + # ({} game left) + else: + rows.insert(0,row) + Logger.info('returned False??, back in queue.'.format(len(rows))) # ({} game left) - except Exception as e: - traceback.print_exc(limit=2, file=sys.stdout) + except WebDriverException: + rows.insert(0,row) + Logger.info('Exception occurred, back in queue.'.format(len(rows))) # ({} left) + Logger.info('WebDriverException occurred, restarting browser.') + raise + + except Exception as e: + traceback.print_exc(limit=2, file=sys.stdout) - Logger.info('Exception occurred, back in queue.') - rows.insert(0,row) + rows.insert(0,row) + Logger.info('Exception occurred, back in queue.'.format(len(rows))) # ({} left) - #badrows.append(row) - # for row in rows: - # try: - # game_page.update(row) - # except: - # badrows.append(row) - # print - Logger.info("Import has finished.") + #badrows.append(row) + # for row in rows: + # try: + # game_page.update(row) + # except: + # badrows.append(row) + # print + except WebDriverException: + pass + Logger.info("Import has finished.") diff --git a/bggcli/main.py b/bggcli/main.py index 79e1203..20eadbc 100644 --- a/bggcli/main.py +++ b/bggcli/main.py @@ -134,11 +134,11 @@ def execute_command(command, argv): try: command_module = import_command_module(command) - print('command_module',command_module) + #print('command_module',command_module) command_args, command_args_options = parse_commad_args(command_module, argv) if command_args: - print(command_args, command_args_options) + #print(command_args, command_args_options) command_module.execute(command_args, command_args_options) show_duration(timer_start) except ImportError: diff --git a/bggcli/version.py b/bggcli/version.py index bb8bac8..c073460 100644 --- a/bggcli/version.py +++ b/bggcli/version.py @@ -1 +1 @@ -VERSION = '0.9.2' +VERSION = '1.-' diff --git a/setup.py b/setup.py index bb77cc9..95943dd 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,8 @@ def run_tests(self): long_description = f.read() setup( - author='Sylvain Francois', - author_email='syllant@gmail.com', + author='Greg Smith , Sylvain Francois ', + author_email='ecomputerd@yahoo.com', classifiers=[ "Development Status :: 4 - Beta", "Environment :: Console", @@ -30,7 +30,7 @@ def run_tests(self): "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", "Topic :: Games/Entertainment :: Board Games" ], cmdclass={'test': PyTest}, From 6eeb10571ea30c791eb7c19b5cb9c428daa23320 Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Wed, 14 Feb 2018 17:27:39 -0600 Subject: [PATCH 5/6] Fixed collection-export command. Updated it to Python 3.6, properly handling unicode. --- README.rst | 4 +++- bggcli/commands/collection_export.py | 26 ++++++++++++++++++-------- bggcli/util/xmltocsv.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 63d47c3..37b6f90 100644 --- a/README.rst +++ b/README.rst @@ -14,11 +14,13 @@ Originally written by Sylvain Francois and updated for Python 3.6 and the 2018 B Only 3 operations are implemented at this time: Updated for BGG 2018 + * bulk import/update for your game collection from a CSV file +* bulk export as a CSV file, WITH version information (game's version is missing in the default export) Not tested with BGG 2018 + * bulk delete from a CSV file -* bulk export as a CSV file, WITH version information (game's version is missing in the default export) Warning: diff --git a/bggcli/commands/collection_export.py b/bggcli/commands/collection_export.py index d66c44e..44bdcb4 100644 --- a/bggcli/commands/collection_export.py +++ b/bggcli/commands/collection_export.py @@ -22,7 +22,12 @@ The CSV file to generate """ import csv -import urllib2 +import codecs +try: + from urllib2 import quote, Request, urlopen +except: + from urllib.request import Request, urlopen + from urllib.parse import quote import time import sys import xml.etree.ElementTree as ET @@ -57,8 +62,8 @@ def execute(args, options): # Use XML2 API, see https://www.boardgamegeek.com/wiki/page/BGG_XML_API2#Collection # Default CSV export doesn't provide version info! url = '%s/xmlapi2/collection?username=%s&version=1&showprivate=1&stats=1' \ - % (BGG_BASE_URL, urllib2.quote(login)) - req = urllib2.Request(url, None, {'Cookie': '%s=%s' % (BGG_SESSION_COOKIE_NAME, auth_cookie)}) + % (BGG_BASE_URL, quote(login)) + req = Request(url, None, {'Cookie': '%s=%s' % (BGG_SESSION_COOKIE_NAME, auth_cookie)}) # Get a BadStatusLine error most of times without this delay! # Related to Selenium, but in some conditions that I have not identified @@ -75,7 +80,8 @@ def execute(args, options): if xml_file == 'true': xml_file_path = write_xml_file(response, dest_path) Logger.info("XML file save as %s" % xml_file_path) - source = open(xml_file_path, 'rU') + #source = open(xml_file_path, 'rU') + source = codecs.open(xml_file_path, mode='rb', encoding='utf-8', errors='replace') else: source = response @@ -93,7 +99,7 @@ def execute(args, options): def default_export(req): - response = urllib2.urlopen(req) + response = urlopen(req) if response.code == 202: Logger.info('Export is queued, will retry in %ss' % EXPORT_QUERY_INTERVAL) @@ -105,7 +111,8 @@ def default_export(req): # Write response in a text file otherwise try: - with open(ERROR_FILE_PATH, "wb") as error_file: + #with open(ERROR_FILE_PATH, "wb") as error_file: + with codecs.open(ERROR_FILE_PATH, mode='wb', encoding='utf-8', errors='replace') as error_file: error_file.write(response.read()) Logger.error("Unexpected response, content has been written in %s" % ERROR_FILE_PATH) except Exception as e: @@ -117,14 +124,17 @@ def default_export(req): def write_xml_file(response, csv_dest_path): dest_path = '.'.join(csv_dest_path.split('.')[:-1]) + '.xml' - with open(dest_path, "wb") as dest_file: + #with open(dest_path, "wb") as dest_file: + with codecs.open(dest_path, mode='wb', encoding='utf-8', errors='replace') as dest_file: dest_file.write(response.read()) return dest_path def write_csv(source, dest_path): - with open(dest_path, "wb") as dest_file: + #with open(dest_path, "wb") as dest_file: + with codecs.open(dest_path, mode='w', encoding='utf-8', errors='replace') as dest_file: + BGG_SUPPORTED_FIELDS.extend(['pp_currency','cv_currency']) csv_writer = csv.DictWriter(dest_file, fieldnames=BGG_SUPPORTED_FIELDS, quoting=csv.QUOTE_ALL) # csv_writer.writeheader() use quotes diff --git a/bggcli/util/xmltocsv.py b/bggcli/util/xmltocsv.py index 42e3877..78e2aea 100644 --- a/bggcli/util/xmltocsv.py +++ b/bggcli/util/xmltocsv.py @@ -37,7 +37,7 @@ def _wrap(node): def _to_str(value): if value is None: return "" - return value.encode('utf8') + return str(value) #.encode('utf8') @staticmethod def _to_int(value): From 221aedf2b4d0bc60a27b4863e696beca10425e17 Mon Sep 17 00:00:00 2001 From: Greg Smith Date: Sun, 18 Feb 2018 18:53:03 -0600 Subject: [PATCH 6/6] Attempt to set name to "bggcli[2018]" for PyPl registration and pip installation. --- README.rst | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 37b6f90..fe55763 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,10 @@ Installation ============ Python 3.6 is required. +:: + + pip install bggcli[2018] + Usage ===== diff --git a/setup.py b/setup.py index 95943dd..b0945e2 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def run_tests(self): keywords='bgg boardgamegeek', license='MIT', long_description=long_description, - name='bggcli', + name='bggcli[2018]', packages=find_packages(), py_modules=['bggcli'], tests_require=["pytest"],