From a88cd872f9aa9c8f410ea9a3d3ccf8e4fc9355ae Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 2 Dec 2010 12:38:26 -0200 Subject: [PATCH 01/31] sets encoding to utf-8 --- lib/ofx/.client.py.swp | Bin 0 -> 16384 bytes lib/ofx/.parser.py.swp | Bin 0 -> 20480 bytes lib/ofx/__init__.py | 7 ++-- lib/ofx/account.py | 7 ++-- lib/ofx/builder.py | 7 ++-- lib/ofx/client.py | 7 ++-- lib/ofx/document.py | 7 ++-- lib/ofx/error.py | 15 +++---- lib/ofx/filetyper.py | 39 +++++++++--------- lib/ofx/generator.py | 55 +++++++++++++------------ lib/ofx/institution.py | 7 ++-- lib/ofx/parser.py | 36 ++++++++-------- lib/ofx/request.py | 31 +++++++------- lib/ofx/response.py | 71 ++++++++++++++++---------------- lib/ofx/validators.py | 21 +++++----- lib/ofxtools/.ofc_parser.py.swp | Bin 0 -> 12288 bytes lib/ofxtools/.qif_parser.py.swp | Bin 0 -> 16384 bytes lib/ofxtools/__init__.py | 15 +++---- lib/ofxtools/csv_converter.py | 7 ++-- lib/ofxtools/ofc_converter.py | 8 ++-- lib/ofxtools/ofc_parser.py | 20 +++++---- lib/ofxtools/ofx_statement.py | 7 ++-- lib/ofxtools/qif_converter.py | 7 ++-- lib/ofxtools/qif_parser.py | 61 +++++++++++++-------------- 24 files changed, 229 insertions(+), 206 deletions(-) create mode 100644 lib/ofx/.client.py.swp create mode 100644 lib/ofx/.parser.py.swp create mode 100644 lib/ofxtools/.ofc_parser.py.swp create mode 100644 lib/ofxtools/.qif_parser.py.swp diff --git a/lib/ofx/.client.py.swp b/lib/ofx/.client.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..24b0e74604d9a99987ce492e5ea8c57c17ff5d9c GIT binary patch literal 16384 zcmeHO-H#hr6(3r@AruiIkWi)5v8vrjUaxmcOOvt^j^k{tWWAB?-6Tp|>w4ze9(FwA z%-mVWLP{y35uhRg;tla3sF3(-UZMX$p?o~>Pzhe(2_Zxgkw83A_? z9gb^L9A{IRiNZIRiNZIRiNZIRiNZIRpO( z45-n)#?!Flp2U{($?sbZ{QfIBHk0dH2kyU?+}Cvk-23W*`^jnklQWPrkTZ}okTZ}o zkTZ}okTZ}okTZ}okTZ}o@Lym+*oJWl{XWkde!Txr_y4!fTw^bfGfZSpaOgXSO@<6UbF$a!2Q7Q-(wiR16~3? z3%I}sfxo}oF#ZO-4BP-JzyrXq-(?t|2R;XE0!zRv?=*}r0?z?WpaA^y9fole@PPY( zm)>p|KLG;ZDzFax^=;4_xB>KlW5AE!3Oj%;;MTo{@pa%d@G6QVKL=g}o&gHL%ixW< z`%YRgc@um{nL}Q)|k0{lGzz>RKOC=oll#ywq{|uyx0*^xJ_DY>ldUhv}S0fp! zWno!#KSXD>lau#eAt;6xoTg7$^$*c2+EVk(ddko*%?2My^X*HKI0D)?Gl@A#89=$HC2M0qsi$8-?$h$v1~PsRyG3=iODG9G7+W}l^P95^_# zEI7Isxh2yyr&6+>fxWTc5C}YIruRsMU_&!t`&OjsaC%c2kk`VTFm03en!2U$>`PA% z-X)j|BPf|Qv;h1IOM$Me5DP5;`po8k;wd%QgV{?K0R%I7AS|&X))o+oV@j&2l zfGIe2$&su*tc+H=T4|oiD4iiBbx(=;<)wq~v0s*B$E5Kw)8lo8#_i|k*STcfVLncX z={R1`U(SeK7`9X1(_yk`0H;RJNsjHloBYhM_)pm0b2_8I?>Vk~V=U)rJaD9o{e~$# z-S7V&IXeC{6)UGAN3j&Wobh71Rw}Ra48eb|m&$y7D3ro_Fk4zI?P zK60`kvJ_9}lB)U>Sn|7{l$I(HHGHysujllmKycbd{)m4;;E#EE(>HJ}q&VSXJ>ms84&itSjLA&< zNYWC{2V-U8+=Fw&04pf3nOI77K7wTjNzMpDk;eFewr~IiTr9s#Dz{Ek6*epFA~lZ_ z58;JFhq;Gbp|l8a+>nZM%G8Ox;#vN17nA^?f9L9(O;IU^5Kaf9zpM zxma&MzP;0?i!4h3p z!lJQWlKVk2NF9&Hfp$A~Gk#BvG2XN<2**IF(W%wd@pxPk`2Kt#OMcK_g)C+;Ts>c} z)*7wa3WlzJn=ELDNEzWQPN);!387V>Nz&5u5e$e~HGpwE#U{f8X(@*p z)E1m6>KJ3sWSp1$nI)>Cbxa!-X%~q)k9(e4micirdB{BRm{zz7{Q{>yP_$kyKfJPD zUMtf@8Hz48V0F)eODvy*?iA?{f(_pP{|NiwFJphq`~UR){VUk#-vnL&ZUD~$8^Fhb zW56x!`ELSOfe~;L_z=K#fX@KW01co7`~iFXZvx){ZUD~$Tfn`*i`eIX7kC=D4m<+f z1N;{I{NDi20R!Ms;8pDNp9h`L1KTn2e_*VZycqylll4vKOK8yg|sawPps zWfrK)u4Y*fYqsJlsQFkd)m)1&+d>JxAB|R&zk)M~5+PN*2{Lc5KMM%0%6uSHx6kM( zSXs-&Kq2LxgVn@NJ0Vy<&zs|TTg!X)jHYcJ_1gM)%c$cm(R5`E)Q7$<3rETR7ByNe zk;}6)C6m5VH9K9`$2$%Yo^dRd)YJ$^0ZPPs;#qJND2)gd07T>}uHWN555YA&(h||h zMce0hD)Jh(WpFw-$QXm_1E!3)#BpJAA=#6oOO_fJA@Gn6Sqwa`JP5=P48#nA7ZX~N zu0rSytgb>vVNw;2)rbw_msZp1Dy>qmeM#{vHFCxe-_+Hb&YWm2J96kIU14A+#kXug zXMD_J_~3Gbw7GJ5#cb6#8#@=aTAQtAYj@3j@`_ohR@?Q)+3j|-(QNUBR=d*P!C!`c zIQi#5-X!q*ZGRL}~1m=BTPX%d^qqbutA(iPB`lJmo4Rm<8VIG(^oZd@@ cgVRTqH63uXbuDcMY4HoEc=A|Iub(jf4cMfpDF6Tf literal 0 HcmV?d00001 diff --git a/lib/ofx/.parser.py.swp b/lib/ofx/.parser.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..c5b0a69096497ba7df0678b2e9a5337fdf41afc7 GIT binary patch literal 20480 zcmeHOYm6jS6)t2^1Q8U1_#hs8U8j4er)PFtaWl>i>7563*zI1odl;6TWxQQ|r@MA) zs)~E7x2Itjh{i}_Gz1f4k2tW9z{_ue@#>A)*;v0XMsKJmJBYx*rb#=|m z4y*=)Kvnje>Z&^T_1$w%)w$U85&Cdx(!z0tWxeC!=MO(}&0W?5S6bF-;kBgF zzQ@mA&p|n(`pua

1^fe>Addi_1tc&HybX93q&*IN95@Ty z0$dCH9-Asp0FMLr0}H_4v9aHPIar&0IJiQ8sQjuvK?_1@`m z+&hSSg_)70+;CWaAieR_*usu54D+DK_PtPg!zmk%Laqm4fBPs1N76_<5*}q~kov+I zFifb|&hxlSLeDvNm@g!Sa2EqHWspR(()HFnzvrz8+xEL&xDxgQX-b}}K*Yu=lvxql zcAS?GY`OxHtx#xx^(918lg8cNx$zV)n*>>x4F@f1`F3|b%_`x*gdAmVNEkJN_6yZE zVn#V*Dg}i~Vo`*uYeP#Uv?NecqcRX`I>j?&+S#OB+=6+`Klq62J`s3hN(u9j04 z{~8>d1&h%tNEcy2!wzm@;%-oWH^}Fv$8$7FOTKF5g&TB)*In1Rkt^j2e462>w@T&L z)Dd6F{89&tIA16fpz{HbNT)QyS*5&bp@n5K=ogO2xyg1@XhhoYl2+rd(; z`eDU`rZ1~yFIO1$a+`c?&5S<8ny?_S(1t~}_osL+CeS4_Ri>tMl*aP~U-bkQ-FCX6 zqbas`ld(1`rOJsWZ<_XGS_f)zkjwX?qBU#H+Th?;Y*`0LDA_(k7U3Rc(cMhQr01l?(nPAYZuA&?IKD9ivQY_HdotQJi;G#vW0eNY$k3k9~DM@`Osg} zc)=fbG+yvHl1|jOnXYsfG@HAqA6_ffAFhr&l*}@v4#?41GLDkMr(G+b!^$kugc%jL zpOR004u)Pb-xr&UK-cyaH@`Rs<>FzQ?wNXbV28PZ_Hg&`E5D~z+|w%XtPIK6V%^hJ zB%?Ruj`hAs$>PJ$cs&)VzUBlCzewg?iWA@z)+EBv$!IhgtCz-Y(oczcL3O zLb`;KDC#=vPAJTKkGhr^{Nv$g76rBSI>akW6@>T$ZM zQk}ypfYX7VuvX)3OT2cmpdH!b4z-}#%7K@GMX;^M1cQ+lFn$K*%5|#Lb9AU&uhfgE zW6^WDVkK|yItukP`y>LiV2dW~hAVp9rhun`XYeN+k9fVC#ftCN5awQ@7X0I*6NbUm z_;|0^D+wf{cVx*|?QvAb6zcKAmD%}feSU1RG!d0K?ssWj^vQ3A0yPV~K?7W^ZlYO) z5bnp4o*%+J<9WdFRvmN^b^;$9+%el{YJ_J?sNsz zzz2XI!q4vmlfXs5F5nsX^!EaLf#=}Ie+zgFxECk^-$9)M;JVqb&yI|NjDd`SjDd`S zjDd`SjDd`S*C+#UE-c2-{f@VW<3Pp8ba{5R(Kvo|euiTZ@h+J3S~Dl6m%U|mvVhn{ zN@%fqQ?<5KU2dmEw}v-j_mN{n>PTI%_b0>+*O)iphNgKH~cYCrilZ?%j@gBva*vpFnn&=Al+c`>tuv)`J@S=E{#dY##7WXw+NyU23gU%@6vtuwV{TO?X2&T<7%Jtg9 znM38nUF*&_vg5ZJ}{fBg# zoVYDA9;1c)^;4UJnT6|FY|u(H3wsMA#{d5)%+ymC;1B%&L4Ns z1cGEFS>k|6wi+fHq{IEMn#6%*I^$S7KJl`IfmbkLz|6') != -1: return "OFC" - + elif re.search("^:20:", self.text, re.MULTILINE) != None and \ re.search("^\:60F\:", self.text, re.MULTILINE) != None and \ re.search("^-$", self.text, re.MULTILINE) != None: return "MT940" - + elif self.text.startswith('%PDF-'): return "PDF" - + elif self.text.find(' 0: frequencies[fields] = frequencies.get(fields, 0) + 1 rows = rows + 1 - + for fieldcount, frequency in frequencies.items(): percentage = (float(frequency) / float(rows)) * float(100) if fieldcount > 2 and percentage > 80: @@ -106,7 +107,7 @@ def trust(self): return "TSV" except StandardError: pass - + # If we get all the way down here, we don't know what the file type is. return "UNKNOWN" diff --git a/lib/ofx/generator.py b/lib/ofx/generator.py index 7734fab..e9795b5 100644 --- a/lib/ofx/generator.py +++ b/lib/ofx/generator.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,38 +37,38 @@ def __init__(self, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", self.curdef = curdef self.lang = lang self.txns_by_date = {} - - def add_transaction(self, date=None, amount=None, number=None, + + def add_transaction(self, date=None, amount=None, number=None, txid=None, type=None, payee=None, memo=None): - txn = ofx.Transaction(date=date, amount=amount, number=number, + txn = ofx.Transaction(date=date, amount=amount, number=number, txid=txid, type=type, payee=payee, memo=memo) txn_date_list = self.txns_by_date.get(txn.date, []) txn_date_list.append(txn) self.txns_by_date[txn.date] = txn_date_list - + def to_ofx1(self): # Sort transactions and fill in date information. # OFX transactions appear most recent first, and oldest last. self.date_list = self.txns_by_date.keys() self.date_list.sort() self.date_list.reverse() - + self.startdate = self.date_list[-1] - self.enddate = self.date_list[0] + self.enddate = self.date_list[0] if self.stmtdate is None: self.stmtdate = date.today().strftime("%Y%m%d") - + # Generate the OFX statement. return DOCUMENT(self._ofx_header(), OFX(self._ofx_signon(), self._ofx_stmt())) - + def to_str(self): return self.to_ofx1() - + def __str__(self): return self.to_ofx1() - + def _ofx_header(self): return HEADER( OFXHEADER("100"), @@ -139,14 +140,14 @@ def _ofx_availbal(self): def _ofx_txns(self): txns = "" - + for date in self.date_list: txn_list = self.txns_by_date[date] txn_index = len(txn_list) for txn in txn_list: txn_date = txn.date txn_amt = txn.amount - + # Make a synthetic transaction ID using as many # uniqueness guarantors as possible. txn.txid = "%s-%s-%s-%s-%s" % (self.org, self.accttype, @@ -154,19 +155,19 @@ def _ofx_txns(self): txn_amt) txns += txn.to_ofx() txn_index -= 1 - + return BANKTRANLIST( DTSTART(self.startdate), DTEND(self.enddate), txns) - - + + # # ofx.Transaction - clean and format transaction information. # class Transaction: - def __init__(self, date="UNKNOWN", amount="0.00", number=None, + def __init__(self, date="UNKNOWN", amount="0.00", number=None, txid=None, type="UNKNOWN", payee="UNKNOWN", memo=None): self.date = date self.amount = amount @@ -175,28 +176,28 @@ def __init__(self, date="UNKNOWN", amount="0.00", number=None, self.type = type self.payee = payee self.memo = memo - + def to_ofx(self): fields = [] - + if self.type is None: self.type = "DEBIT" - + fields.append(TRNTYPE(self.type)) fields.append(DTPOSTED(self.date)) fields.append(TRNAMT(self.amount)) - + if self.number is not None: fields.append(CHECKNUM(self.number)) - + if self.txid is None: self.txid = uuid.generate().upper() - + fields.append(FITID(self.txid)) fields.append(NAME(self.payee)) - + if self.memo is not None: fields.append(MEMO(self.memo)) return STMTTRN(*fields) - + diff --git a/lib/ofx/institution.py b/lib/ofx/institution.py index de9657b..298b07f 100644 --- a/lib/ofx/institution.py +++ b/lib/ofx/institution.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index 65737cc..5a5e63d 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -1,11 +1,13 @@ +#coding: utf-8 + # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,17 +24,17 @@ Literal, OneOrMore, Optional, SkipTo, White, Word, ZeroOrMore def _ofxStartDebugAction( instring, loc, expr ): - sys.stderr.write("Match %s at loc %s (%d,%d)" % - (expr, loc, - instring.count("\n", 0, loc) + 1, + sys.stderr.write("Match %s at loc %s (%d,%d)" % + (expr, loc, + instring.count("\n", 0, loc) + 1, loc - instring.rfind("\n", 0, loc))) def _ofxSuccessDebugAction( instring, startloc, endloc, expr, toks ): sys.stderr.write("Matched %s -> %s" % (expr, str(toks.asList()))) - + def _ofxExceptionDebugAction( instring, loc, expr, exc ): sys.stderr.write("Exception raised: %s" % exc) - + class Parser: """Dirt-simple OFX parser for interpreting server results (primarily for errors at this point). Currently parses OFX 1.02.""" @@ -41,7 +43,7 @@ def __init__(self, debug=False): header = Group(Word(alphas) + Literal(":").suppress() + Optional(CharsNotIn("\r\n"))) headers = Dict(OneOrMore(header)).setResultsName("header") - + # Parser definition for OFX body aggregate = Forward().setResultsName("OFX") aggregate_open_tag, aggregate_close_tag = self._tag() @@ -51,12 +53,12 @@ def __init__(self, debug=False): + Dict(ZeroOrMore(aggregate | content)) \ + aggregate_close_tag) body = Group(aggregate).setResultsName("body") - + # The parser as a whole self.parser = headers + body if (debug): self.parser.setDebugActions(_ofxStartDebugAction, _ofxSuccessDebugAction, _ofxExceptionDebugAction) - + def _tag(self, closed=True): """Generate parser definitions for OFX tags.""" openTag = Literal("<").suppress() + Word(alphanums + ".") \ @@ -66,7 +68,7 @@ def _tag(self, closed=True): return openTag, closeTag else: return openTag - + def parse(self, ofx): """Parse a string argument and return a tree structure representing the parsed document.""" @@ -76,7 +78,7 @@ def parse(self, ofx): ofx = self.strip_junk_ascii(ofx) ofx = self.fix_unknown_account_type(ofx) return self.parser.parseString(ofx).asDict() - + def strip_empty_tags(self, ofx): """Strips open/close tags that have no content.""" strip_search = '<(?P[^>]+)>\s*' @@ -89,16 +91,16 @@ def strip_close_tags(self, ofx): it from scratch just yet.""" strip_search = '<(?P[^>]+)>\s*(?P[^<\n\r]+)(?:\s*)?(?P[\n\r]*)' return re.sub(strip_search, '<\g>\g\g', ofx) - + def strip_blank_dtasof(self, ofx): """Strips empty dtasof tags from wells fargo/wachovia downloads. Again, it would be better to just rewrite the parser, but for now this is a workaround.""" blank_search = '<(DTASOF|BALAMT|BANKID|CATEGORY|NAME)>[\n\r]+' return re.sub(blank_search, '', ofx) - + def strip_junk_ascii(self, ofx): - """Strips high ascii gibberish characters from Schwab statements. They seem to - contains strings of EF BF BD EF BF BD 0A 08 EF BF BD 64 EF BF BD in the field, + """Strips high ascii gibberish characters from Schwab statements. They seem to + contains strings of EF BF BD EF BF BD 0A 08 EF BF BD 64 EF BF BD in the field, and the newline is screwing up the parser.""" return re.sub('[\xBD-\xFF\x64\x0A\x08]{4,}', '', ofx) diff --git a/lib/ofx/request.py b/lib/ofx/request.py index d82fb3b..00a0125 100644 --- a/lib/ofx/request.py +++ b/lib/ofx/request.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,19 +30,19 @@ def __init__(self, cookie=4, app_name="Money", app_version="1400"): self.app_version = app_version self.cookie = cookie # FIXME: find out the meaning of this magic value. Why not 3 or 5? self.request_id = str(uuid.uuid4()).upper() - + def _format_date(self, date=None, datetime=datetime.datetime.now()): if date == None: return datetime.strftime("%Y%m%d%H%M%S") else: return date.strftime("%Y%m%d") - + def _message(self, institution, username, password, body): """Composes a complete OFX message document.""" return DOCUMENT(self._header(), OFX(self._sign_on(institution, username, password), body)) - + def _header(self): """Formats an OFX message header.""" return HEADER( @@ -54,7 +55,7 @@ def _header(self): COMPRESSION("NONE"), OLDFILEUID("NONE"), NEWFILEUID(self.request_id)) - + def _sign_on(self, institution, username, password): """Formats an OFX sign-on block.""" return SIGNONMSGSRQV1( @@ -68,7 +69,7 @@ def _sign_on(self, institution, username, password): FID(institution.ofx_fid)), APPID(self.app_name), APPVER(self.app_version))) - + def fi_profile(self, institution, username, password): return self._message(institution, username, password, PROFMSGSRQV1( @@ -78,7 +79,7 @@ def fi_profile(self, institution, username, password): PROFRQ( CLIENTROUTING("NONE"), DTPROFUP("19980101"))))) - + def account_info(self, institution, username, password): """Returns a complete OFX account information request document.""" return self._message(institution, username, password, @@ -88,7 +89,7 @@ def account_info(self, institution, username, password): CLTCOOKIE(self.cookie), ACCTINFORQ( DTACCTUP("19980101"))))) - + def bank_stmt(self, account, username, password, daysago=90): """Returns a complete OFX bank statement request document.""" dt_start = datetime.datetime.now() - datetime.timedelta(days=daysago) @@ -105,7 +106,7 @@ def bank_stmt(self, account, username, password, daysago=90): INCTRAN( DTSTART(self._format_date(date=dt_start)), INCLUDE("Y")))))) - + def bank_closing(self, account, username, password): """Returns a complete OFX bank closing information request document.""" return self._message(account.institution, username, password, @@ -118,7 +119,7 @@ def bank_closing(self, account, username, password): BANKID(account.aba_number), ACCTID(account.acct_number), ACCTTYPE(account.get_ofx_accttype())))))) - + def creditcard_stmt(self, account, username, password, daysago=90): """Returns a complete OFX credit card statement request document.""" dt_start = datetime.datetime.now() - datetime.timedelta(days=daysago) @@ -133,7 +134,7 @@ def creditcard_stmt(self, account, username, password, daysago=90): INCTRAN( DTSTART(self._format_date(date=dt_start)), INCLUDE("Y")))))) - + def creditcard_closing(self, account, username, password): """Returns a complete OFX credit card closing information request document.""" dt_start = datetime.datetime.now() - datetime.timedelta(days=61) @@ -148,5 +149,5 @@ def creditcard_closing(self, account, username, password): ACCTID(account.acct_number)), DTSTART(self._format_date(date=dt_end)), DTEND(self._format_date(date=dt_end)))))) - - + + diff --git a/lib/ofx/response.py b/lib/ofx/response.py index b0887d1..0c077b8 100644 --- a/lib/ofx/response.py +++ b/lib/ofx/response.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,31 +28,31 @@ def __init__(self, response, debug=False): # B of A is mechanized. # REVIEW: Checked. Still needed. Feh! self.raw_response = response.replace('Content- type:application/ofx', "") - + # Good god, another one. Regex? self.raw_response = self.raw_response.replace('Content-Type: application/x-ofx', "") - + # I'm seeing this a lot, so here's an ugly workaround. I wonder why multiple # FIs are causing it, though. self.raw_response = self.raw_response.replace('****OFX download terminated due to exception: Null or zero length FITID****', '') - + parser = ofx.Parser(debug) self.parse_dict = parser.parse(self.raw_response) self.ofx = self.parse_dict["body"]["OFX"].asDict() - + def as_dict(self): return self.ofx - + def as_string(self): return self.raw_response - + def get_encoding(self): return self.parse_dict["header"]["ENCODING"] - + def get_statements(self): # This allows us to parse out all statements from an OFX file # that contains multiple statements. - + # FIXME: I'm not positive this is legitimate. Are there tagsets # a bank might use inside a bank or creditcard response *other* # than statements? I bet there are. @@ -61,7 +62,7 @@ def get_statements(self): for sub_tag in self.ofx[tag]: statements.append(ofx.Statement(sub_tag)) return statements - + def get_accounts(self): accounts = [] for tag in self.ofx.keys(): @@ -79,45 +80,45 @@ def get_accounts(self): if account is not None: accounts.append(account) return accounts - + def _extract_account(self, acct_block): acct_dict = acct_block.asDict() - + if acct_dict.has_key("DESC"): desc = acct_dict["DESC"] else: desc = None - + if acct_dict.has_key("BANKACCTINFO"): acctinfo = acct_dict["BANKACCTINFO"] return ofx.Account(ofx_block=acctinfo["BANKACCTFROM"], desc=desc) - + elif acct_dict.has_key("CCACCTINFO"): acctinfo = acct_dict["CCACCTINFO"] account = ofx.Account(ofx_block=acctinfo["CCACCTFROM"], desc=desc) account.acct_type = "CREDITCARD" return account - + else: return None - + def check_signon_status(self): status = self.ofx["SIGNONMSGSRSV1"]["SONRS"]["STATUS"] # This will throw an ofx.Error if the signon did not succeed. self._check_status(status, "signon") # If no exception was thrown, the signon succeeded. return True - + def _check_status(self, status_block, description): # Convert the PyParsing result object into a dictionary so we can # provide default values if the status values don't exist in the # response. status = status_block.asDict() - + # There is no OFX status code "-1," so I'm using that code as a # marker for "No status code was returned." code = status.get("CODE", "-1") - + # Code "0" is "Success"; code "1" is "data is up-to-date." Anything # else represents an error. if code is not "0" and code is not "1": @@ -127,19 +128,19 @@ def _check_status(self, status_block, description): # information from the bank. severity = status.get("SEVERITY", "NONE") message = status.get("MESSAGE", "NONE") - + # The "description" allows the code to give some indication # of where the error originated (for instance, the kind of # account we were trying to download when the error occurred). error = ofx.Error(description, code, severity, message) raise error - + class Statement(ofx.Document): def __init__(self, statement): self.parse_result = statement self.parse_dict = self.parse_result.asDict() - + if self.parse_dict.has_key("STMTRS"): stmt = self.parse_dict["STMTRS"] self.account = ofx.Account(ofx_block=stmt["BANKACCTFROM"]) @@ -150,39 +151,39 @@ def __init__(self, statement): else: error = ValueError("Unknown statement type: %s." % statement) raise error - + self.currency = self._get(stmt, "CURDEF") self.begin_date = self._get(stmt["BANKTRANLIST"], "DTSTART") self.end_date = self._get(stmt["BANKTRANLIST"], "DTEND") self.balance = self._get(stmt["LEDGERBAL"], "BALAMT") self.bal_date = self._get(stmt["LEDGERBAL"], "DTASOF") - + def _get(self, data, key): data_dict = data.asDict() return data_dict.get(key, "NONE") - + def as_dict(self): return self.parse_dict - + def as_xml(self, indent=4): taglist = self.parse_result.asList() return self._format_xml(taglist, indent) - + def get_account(self): return self.account - + def get_currency(self): return self.currency - + def get_begin_date(self): return self.begin_date - + def get_end_date(self): return self.end_date - + def get_balance(self): return self.balance - + def get_balance_date(self): return self.bal_date - + diff --git a/lib/ofx/validators.py b/lib/ofx/validators.py index 6e0e62a..4087318 100644 --- a/lib/ofx/validators.py +++ b/lib/ofx/validators.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,11 +30,11 @@ def __init__(self, number): self.digits = None self.region_code = None self.converted = False - + def is_valid(self): if self.converted is False or len(self.digits) != 9: return False - + checksum = ((self.digits[0] * 3) + (self.digits[1] * 7) + self.digits[2] + @@ -44,7 +45,7 @@ def is_valid(self): (self.digits[7] * 7) + self.digits[8] ) return (checksum % 10 == 0) - + def get_type(self): # Remember that range() stops one short of the second argument. # In other words, "x in range(1, 13)" means "x >= 1 and x < 13". @@ -60,7 +61,7 @@ def get_type(self): return "Traveller's Cheque" else: return None - + def get_region(self): if self.region_code == 0: return "United States Government" @@ -92,11 +93,11 @@ def get_region(self): return "Traveller's Cheque" else: return None - + def to_s(self): return str(self.number) + " (valid: %s; type: %s; region: %s)" % \ (self.is_valid(), self.get_type(), self.get_region()) - + def __repr__(self): return self.to_s() - + diff --git a/lib/ofxtools/.ofc_parser.py.swp b/lib/ofxtools/.ofc_parser.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..374cb9838f45899925027000b7b8653eb06d41b4 GIT binary patch literal 12288 zcmeHNy^kA36rWJ=9fFdEAS7O##@6kvFBeeYxI;SbBUoH)biU*WTx5;cV|$hL&N4F_ z-$^J)6i9URsi;6ilvF@V11b0y0144Sg;Id{&3bL`eIyZO(5&>+?#{e<^WN`$tSnD^ z?n>XJ0wa&d;Bl zvyx{^XXw<)%MV!OrX2&00mp!2z%k$$a11yG90QJl{}TfyImSMOLyzQ6y_(;T9k}PO z?&cV93^)cH1C9a5fMdWh;23ZWI0hU8jseHOL&yN{;D7!{fyZDx{{J8U{{Qm?V}Ag5 zfFFQQfOX&n;CbNpXBhhp_zbuPoCJP6&e(0>OW+Hj2MC}8Yyl^LU!G>{XW%E`Q{Xyq z0l51VW8VWG16P1Ez&}qi_BZei@Hy}va2)vV3C6w!ZUF)CB5?O{#(o7p0q*SF`)j+i-OC^|Tn z>AQ#0gR=MC-=1@`$K9Gng>kQp<8Kw8>(wwY<;q0A=?luFI@FOtIuNn(OQq6e9?x%E zCB#W*dWfrDCPp~YM>!3gj1t01qycyl7*CYYP==i|T^-RK?Ninw>Uw0YS(v&*xN==G z5%Crut+K-+(qe(xr}4F4(4!!dS`?R9!SNt1(M2WGq+D98mS~2yrRtP2rw9axacDvrYwAixjrDiP@SdJT(?+0$kvsq+ z>zB4vDh>vs=+D9D@o7rGHGW##4Mbx1T@~%Lw;b5VX?`7CfAD>*WvZ{41_8(}X!BTc zmCc_~Vk=(@Q;^fh@zGh&f;PidQD&;khOlEzE<~`Pdy+T2*HfYgrV&kHHib?jqw9Pi z%B69|lq&l(d4?v7S{cul_x+sAP*#YISmfz8SwSCAAY+5tsZ?kh^6C8IVxG~Hp_a`l zyGKDj*D-M;f1irZ&`KYbU6-a7qd0cdPF$)?jXWg3gk}TgvlPmcXPu)qr%Mjo;~hl= zOqyysVm0R1r5bY7>0!zn>nmW}=0ldxEt~VLR-V&YIG09Njmi>*vKlHgtHVJO2^7W3 zs0*pAMncf6P};UjWpm?zj;bbzxYpF@23ZGGXjgf~dNhffM_Hf14SmHc^bgimazM$R zZLU^543Nsur1N+M`n@a}ys_KfD83$|nXek>H4a;{vxMfEr}^|qGtG`28(D2zfD*iO z*fXerjwl?2hL74sYR_hq{Ss`1frz#6A<#rWFAQaT*>|0YAnb+#MLh1MyeFt9ZwM9J zjS`^-q1IUs!q)*>wrg8&HZE_`_VVWDa(%0|)}+QJtu*SZwXIsCj<~a` z)&gSQPeTPg?LFcNb_jSI*%9%f-IJr1q)K=E&&9&Kie=b{cRi-rHdnDV2LoY{>CwS}bBbgf*xtB}E z(ol*ouj#|NZU_TK)@rR;fwaW%I1n@hX+`2F*Quf(F!P1`N?FiQ}S*f_oN^QRHwD+Tf#hWEuO2@H8=^AfuyM`y2 zS9JSR>P>H_45SPk!GJH@GoL#}K0WvFtSb3n<}5w^k&PpS(#I(SDFZ13DFZ13DFZ13 zDFZ13DFgpc3`qAb@+CCruCP(jj^JJ4b##1pc>Tzfa^datkTQ@mkTQ@mkTQ@mkTQ@m zkTQ@mkTQ@mkTQ@ma0fDAv6P1TmeeJf8Gb%fE$1XJO2 zUIU&67Jz$!dw^T-A>;?ZHQ*6|06)K*kZ%BA2N+NSiooxY6!{(SZJ-8x9QZ8~DYt-c z0$%_=3j7A#YJQXSKYb^DWO^m}qF{233Ez^UX7pGtQ!zw$-7{=qn36lT$m9mF!FZYySR=c>(CF_Zt?bzJjWkU9tErWac!a%(V%jCW-C_PUv zdXDerU@Nnj5X%tv3@evCyWV%%ylO%gE%RO1!_!=zo`cIx{Y`SPVAwl%CI?MemWTt@ zgsB{a{RBp)Gg;StIiCf26fDYmdzsrT&gx;D24rU!vvi(j7MU6qH7%vwF;hsOWK9=eH})H5m$iJ0#gz3cPzQ}sBvq|g1S}`U6ZU2TV{qf$9caU(jN-! zQWHg+c_D$Z&YhqxypdfV6*SGH)Yr9O#p(695SW>gFtMU#R#?wTQuP$WS2Z{s*y9T3!?kk%UVH$nZB1$K`=7QdS!@d7@xG2 z2JKX#C&K4Ynna4H9kVmqpc?BKlJByHYw>7kCh@GmCF4>FI0y!`7E0bBl_X78XEKtX0byQi$ zLC2yv)d7lO;mj+~MG8DbOv__M9Gd`|1;9Zg(i}xsE64SQuIeqSFFZ0n?HV zJHq%>)ewqAUo_*>OY$J4xGWA@LT4zV1ca_#X1A#9GTOt8ffGYgF6e6YLJ?9Bvs5=? zszj7def4p+F1(D{AHfOA&<11iTeR&2MTE;t-sUESYj=F3!)V9ZWgdd>4t1FqF0g{) zi*MAjSzZ5pePf+&me<4qrd%%sJtaq$9<4FhPV+@RG)hAxyF)kXnzY}hK~M+aT=IlP%R z`_you!8Do{LyPv5PNU;7Jd+OPHr>PWhNcy$aN2SY%XL6HJfbV5J6|g8?d=r}e1G0$ zMaS!uP?us*l$NWNrCMWYX0AA^MJ_u&?HPUQG^K%h4fKI>Bi zXk44aK(=o(7aeKqPPVw3T<9K1i&vt!3i~lmR9V{?Jt@#9@I5|ypTa(xQ;w1eg);d^ zwZRqG=kZQg(z)5$&&>=YTw*_Vj-PUIx0r3{U|6 zggyNC0Snjxioidxe}4hk0v-U)0QUhmv4?j73OoRu0se~p`y0SZz%#&;z^8z>uz$Y^ zd=2<2-~x{UYH$BXwC@jUFAv-RRC_N1=`Up08D*qQ+sR|aTu6NSdK*>bLPM0275Lu#+Ntzc3 z=#Vg%y}GoqMz5;e*=S}oTk$=Q*=Ap*#0I*;)&}{=K{Tl{o2p6M76P7pyu4C1c!Hn< ziKLb;I+`+7MGzPdlcSoykd}dR70<;VsHj3oiX#3Z=utUU^tjTge(Hv{2sJ--g{WHt z@VVC0NgJmD`H1oRDr=S(HDkI14AH!3;3 zfyw0$8;v7+hdyVThS$=`;p^0la^=_QAZ2_V8wR7vk+$foQsZ7ZQY$WEHeuH$wI`3} z)FX))*YK=hD8!@>Xwh*K@)eaTUdC8Dl*gFP;lL5O(UfQG-Gtsjs&);m9-p+L7Ya)K z)NFc0!|*ui3sDy5oo_>!&pj-aAHEckCu%`C*ts940pqWCMmGP^923@CwcAc6AAfTc zIo)9`3x!YAu2!>7A6_2p9cL@5TSw@)5}(G0D?sFBY6u=&*ZgE% G1@d17-_G>_ literal 0 HcmV?d00001 diff --git a/lib/ofxtools/__init__.py b/lib/ofxtools/__init__.py index a21f47d..2e55aec 100644 --- a/lib/ofxtools/__init__.py +++ b/lib/ofxtools/__init__.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,14 +24,14 @@ import sys def _ofxtoolsStartDebugAction( instring, loc, expr ): - sys.stderr.write("Match %s at loc %s (%d,%d)" % - (expr, loc, - instring.count("\n", 0, loc) + 1, + sys.stderr.write("Match %s at loc %s (%d,%d)" % + (expr, loc, + instring.count("\n", 0, loc) + 1, loc - instring.rfind("\n", 0, loc))) def _ofxtoolsSuccessDebugAction( instring, startloc, endloc, expr, toks ): sys.stderr.write("Matched %s -> %s" % (expr, str(toks.asList()))) - + def _ofxtoolsExceptionDebugAction( instring, loc, expr, exc ): sys.stderr.write("Exception raised: %s" % exc) diff --git a/lib/ofxtools/csv_converter.py b/lib/ofxtools/csv_converter.py index 9f060d4..af9f332 100644 --- a/lib/ofxtools/csv_converter.py +++ b/lib/ofxtools/csv_converter.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib/ofxtools/ofc_converter.py b/lib/ofxtools/ofc_converter.py index 8a2ca62..52ce57a 100644 --- a/lib/ofxtools/ofc_converter.py +++ b/lib/ofxtools/ofc_converter.py @@ -1,11 +1,13 @@ +#coding: utf-8 + # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 78d5983..46e7673 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -1,11 +1,13 @@ +#coding: utf-8 + # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,13 +32,13 @@ def __init__(self, debug=False): aggregate << Group(aggregate_open_tag \ + Dict(OneOrMore(aggregate | content)) \ + aggregate_close_tag) - + self.parser = Group(aggregate).setResultsName("document") if (debug): - self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, - ofxtools._ofxtoolsSuccessDebugAction, + self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, + ofxtools._ofxtoolsSuccessDebugAction, ofxtools._ofxtoolsExceptionDebugAction) - + def _tag(self, closed=True): """Generate parser definitions for OFX tags.""" openTag = Literal("<").suppress() + Word(alphanums + ".") \ @@ -46,10 +48,10 @@ def _tag(self, closed=True): return openTag, closeTag else: return openTag - + def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" return self.parser.parseString(ofc).asDict() - + diff --git a/lib/ofxtools/ofx_statement.py b/lib/ofxtools/ofx_statement.py index 918607d..b83b2c0 100644 --- a/lib/ofxtools/ofx_statement.py +++ b/lib/ofxtools/ofx_statement.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib/ofxtools/qif_converter.py b/lib/ofxtools/qif_converter.py index 7128957..c9f2d25 100644 --- a/lib/ofxtools/qif_converter.py +++ b/lib/ofxtools/qif_converter.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib/ofxtools/qif_parser.py b/lib/ofxtools/qif_parser.py index 2638c78..3720269 100644 --- a/lib/ofxtools/qif_parser.py +++ b/lib/ofxtools/qif_parser.py @@ -1,11 +1,12 @@ +#coding: utf-8 # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +32,7 @@ def __init__(self, debug=False): 'B' : "Balance", '/' : "BalanceDate", '$' : "Balance" } - + noninvestment_items = { 'D' : "Date", 'T' : "Amount", 'U' : "Amount2", @@ -45,7 +46,7 @@ def __init__(self, debug=False): 'E' : "SplitMemo", '$' : "SplitAmount", '-' : "NegativeSplitAmount" } - + investment_items = { 'D' : "Date", 'N' : "Action", 'Y' : "Security", @@ -58,7 +59,7 @@ def __init__(self, debug=False): 'O' : "Commission", 'L' : "TransferAccount", '$' : "TransferAmount" } - + category_items = { 'N' : "Name", 'D' : "Description", 'T' : "TaxRelated", @@ -66,61 +67,61 @@ def __init__(self, debug=False): 'E' : "ExpenseCategory", 'B' : "BudgetAmount", 'R' : "TaxSchedule" } - + class_items = { 'N' : "Name", 'D' : "Description" } - + options = Group(CaselessLiteral('!Option:') + restOfLine).suppress() - - banktxns = Group(CaselessLiteral('!Type:Bank').suppress() + + + banktxns = Group(CaselessLiteral('!Type:Bank').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("BankTransactions") - - cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() + + + cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CashTransactions") - + ccardtxns = Group(Or([CaselessLiteral('!Type:CCard').suppress(), - CaselessLiteral('!Type!CCard').suppress()]) + + CaselessLiteral('!Type!CCard').suppress()]) + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CreditCardTransactions") - - liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() + + + liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CreditCardTransactions") - - invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() + + + invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() + ZeroOrMore(self._items(investment_items)) ).setResultsName("InvestmentTransactions") - + acctlist = Group(CaselessLiteral('!Account').suppress() + ZeroOrMore(Or([self._items(account_items, name="AccountInfo")])) ).setResultsName("AccountList") - + category = Group(CaselessLiteral('!Type:Cat').suppress() + ZeroOrMore(self._items(category_items)) ).setResultsName("CategoryList") - + classlist = Group(CaselessLiteral('!Type:Class').suppress() + ZeroOrMore(self._items(category_items)) ).setResultsName("ClassList") - + self.parser = Group(ZeroOrMore(White()).suppress() + ZeroOrMore(acctlist).suppress() + OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns) + ZeroOrMore(White()).suppress() ).setResultsName("QifStatement") - + if (debug): - self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, - ofxtools._ofxtoolsSuccessDebugAction, + self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, + ofxtools._ofxtoolsSuccessDebugAction, ofxtools._ofxtoolsExceptionDebugAction) - - + + def _items(self, items, name="Transaction"): item_list = [] for (code, name) in items.iteritems(): @@ -130,12 +131,12 @@ def _items(self, items, name="Transaction"): oneOf('^EUR ^').setResultsName('Currency') + LineEnd().suppress() ).setResultsName(name) - + def _item(self, code, name): return CaselessLiteral(code).suppress() + \ restOfLine.setResultsName(name) + \ LineEnd().suppress() - + def parse(self, qif): return self.parser.parseString(qif) - + From cd3043eb1f30e1b8995eb5757adb88e56dd72df6 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 2 Dec 2010 13:44:40 -0200 Subject: [PATCH 02/31] force xml encoding to utf-8 --- lib/ofx/.document.py.swp | Bin 0 -> 12288 bytes lib/ofx/document.py | 11 ++--------- 2 files changed, 2 insertions(+), 9 deletions(-) create mode 100644 lib/ofx/.document.py.swp diff --git a/lib/ofx/.document.py.swp b/lib/ofx/.document.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..3b18b187f6059d8d790ea486091f2a24528da564 GIT binary patch literal 12288 zcmeHNO>87b74AUzClIkh0)@a!kBrCJneo^WBJ#$GCia+&aK@6y-py)bRyEx%VE+>)n;$0_xuV z+}l<4UcLI>tFL-@d)U6-dyzibTDNdLYFTf*{^suQKXk|X+7-(>5rHcstpe`5$5o7L zx>6q-6<#v-PBCzGrlMmV3P-NCPUPyyJ5{68{N{VZRep4p6HjGeTH*W>hgEaMK*hj= zF%X+k^XW&dr`FfnY*4eGper9Zd@$Q;RK-BWK*d1CK*d1CK*d1CK*d1C!2bmUCVa^H z0#@>tY&HGt{?NI5_E_C21}X+B1}X+B1}X+B1}X+B1}X+B1}X+B1}X;r2N@8qW&I3) z+5hNU5j_6?KmYsx-`;OozXW~`d>Qx@@Hp@o@R#>l)}MfH0-ph{0Ka&|vc3m=7x)hF zMc@wbc_0D?z&`Li@G$Vx_gdDEfFA-gU=8^Ddo1hwfDhDxKfc?tz78A#9|nH+F3Wlq zH~>Bf{N|k)4}1l91qgvBfxo`Pvc3U)7FYxR`F6{C6ZjSIC141A1b7Vi!`m$DC%|jK zw}Ec~Cx8Sxz!LE1hb`;Jz*m8N;Bnwz;Ohs#99RcF1~7;Jz(1t?7I+nifv12+fnS5; z*YbZ<`7hK7WmhCk90jREZnCm=N;~Lu>K9Gig=f0DN%oR{W=TJbpR2gN9N4r(bq}9= z6oW6fe}EzPoFXvy^KifZ`z+#q=I8p)JuWD-{$n-+qfO)m;|3XlA1A_(W%B2hVp77D zw9_VOaoUpF5uvQ}ALAf1G5IMGR{OOA}w#bo?~v=L*>rxqg(dg&P(=DOXyv!y1ufQ zmvH6jx9w-1Xf|nE7)fa|X(sdrEoo{tSx9V_5*OXs+KK=`e5o0PQ=km>RUj7x-({vP z6`aV(NS8*Dnx3CXMc&v8gnyUG&UKxgu!xFhS9eCE>%TaG5F+G?wOeZjYTu*;Ol?lEMlj z4kDn#L1(bl>k*W3`LN&H+S~3fb9Ils6B*!iK`_`gVrk7saEqO?eFHW=PN|9y*(Q?C7X+w^hPE}BZh1rq$^DCLF7#5M5 zA(%NSWF}u*RztACm}M8je=M_1jTLfK3WQZlqUh&PF4CbC$C|nUtL+5DL|j>#vb;3X z%E4NFSx!olDiRu{pC;4BWH*r{UJv%sIy-hyxyON;`O+Q3<+;2L(b&hKAfZYGHYcNjg614@Fr2dWRy&PnBFffC@i7iKE-XP) zsdY9jn=IB~gb#w!-D+pFl(stc_YS%nATX5;&I9Zlg!9RGPV9>a8#irCoHiB;!s1h$ zri2ShwUfMGFj`>g1RcYTChn7;nP^?hnX1kEpac{jdr##fkDjlkb&s|d9Ts_+s4%}S`x z+glilvA#F;OpTBgSfPdk zmLKTD)YGitOp^;QH+lyz>>VD^jn4jlr+?7v4rp(mw)Xnly#pK!c-o;(|0aF9*WYfC z#ET^&k*BP2nq>(L_#UJVu3WrJL>{yS1$rT3R*CWeY+&Tc?hL3mu<7~Epf_m1$6?^J z<4C>|dlCE$=OjYR;E16wv7#BzDaLsFP2>aBGK)NuNWE4gGlLYQOru67#)KQItFzgx zCGh?EM7C5kUWG4aP_ORxwz~a6w~0DQOm0Fka6FJ28o?R{oeA-j3QtZl^tQ&jk&I+{ zNDmh*j>QhdI5LE`1}sttn_n{1xA9V@n5)cRnW#<@W8TxC>*-;eY{@#C+nJHQaxDpX j3-O_HCx&p8f1_GiD7(9Pe literal 0 HcmV?d00001 diff --git a/lib/ofx/document.py b/lib/ofx/document.py index 55b624c..2a32618 100644 --- a/lib/ofx/document.py +++ b/lib/ofx/document.py @@ -41,15 +41,8 @@ def as_xml(self, original_format=None, date_format=None): # method so that we can start to survey what encodings # we're actually seeing, and use that to maybe be smarter # about this in the future. - encoding = "" - if self.parse_dict["header"]["ENCODING"] == "USASCII": - encoding = "US-ASCII" - elif self.parse_dict["header"]["ENCODING"] == "UNICODE": - encoding = "UTF-8" - elif self.parse_dict["header"]["ENCODING"] == "NONE": - encoding = "UTF-8" - else: - encoding = self.parse_dict["header"]["ENCODING"] + #forcing encoding to utf-8 + encoding = "UTF-8" xml += """\n""" % encoding xml += """ Date: Thu, 2 Dec 2010 14:42:29 -0200 Subject: [PATCH 03/31] added string parameter support --- fixofx.py | 75 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/fixofx.py b/fixofx.py index 0ebbb2a..4af1cf7 100755 --- a/fixofx.py +++ b/fixofx.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -57,67 +57,67 @@ def fixpath(filename): pass -def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", +def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", accttype="UNKNOWN", acctid="UNKNOWN", - balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False, + balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False, debug=False): - + # This finishes a verbosity message started by the caller, where the # caller explains the source command-line option and this explains the # source format. - if verbose: + if verbose: sys.stderr.write("Converting from %s format.\n" % filetype) if options.debug and (filetype in ["OFC", "QIF"] or filetype.startswith("OFX")): sys.stderr.write("Starting work on raw text:\n") sys.stderr.write(rawtext + "\n\n") - + if filetype.startswith("OFX/2"): if verbose: sys.stderr.write("No conversion needed; returning unmodified.\n") - + # The file is already OFX 2 -- return it unaltered, ignoring # any of the parameters passed to this method. return text - + elif filetype.startswith("OFX"): if verbose: sys.stderr.write("Converting to OFX/2.0...\n") - + # This will throw a ParseException if it is unable to recognize # the source format. - response = ofx.Response(text, debug=debug) + response = ofx.Response(text, debug=debug) return response.as_xml(original_format=filetype) - + elif filetype == "OFC": if verbose: sys.stderr.write("Beginning OFC conversion...\n") converter = ofxtools.OfcConverter(text, fid=fid, org=org, curdef=curdef, lang=lang, debug=debug) - + # This will throw a ParseException if it is unable to recognize # the source format. - if verbose: + if verbose: sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" % converter.to_ofx102()) sys.stderr.write("Converting to OFX/2.0...\n") - + return converter.to_xml() - + elif filetype == "QIF": if verbose: sys.stderr.write("Beginning QIF conversion...\n") converter = ofxtools.QifConverter(text, fid=fid, org=org, - bankid=bankid, accttype=accttype, - acctid=acctid, balance=balance, + bankid=bankid, accttype=accttype, + acctid=acctid, balance=balance, curdef=curdef, lang=lang, dayfirst=dayfirst, debug=debug) - + # This will throw a ParseException if it is unable to recognize # the source format. - if verbose: + if verbose: sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" % converter.to_ofx102()) sys.stderr.write("Converting to OFX/2.0...\n") - + return converter.to_xml() - + else: raise TypeError("Unable to convert source format '%s'." % filetype) @@ -148,6 +148,8 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", help="(QIF only) Account balance to use in output") parser.add_option("--dayfirst", action="store_true", dest="dayfirst", default=False, help="(QIF only) Parse dates day first (UK format)") +parser.add_option("-s", "--string", dest="string", default=None, + help="string to convert") (options, args) = parser.parse_args() # @@ -168,9 +170,9 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", if options.filename: if os.path.isfile(options.filename): - if options.verbose: + if options.verbose: sys.stderr.write("Reading from '%s'\n." % options.filename) - + try: srcfile = open(options.filename, 'rU') rawtext = srcfile.read() @@ -180,19 +182,24 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", print "Exiting." sys.stderr.write("fixofx failed with error code 1\n") sys.exit(1) - + else: print "'%s' does not appear to be a file. Try --help." % options.filename sys.stderr.write("fixofx failed with error code 2\n") sys.exit(2) +elif options.string: + if options.verbose: + sys.stderr.write("Reading from string\n") + rawtext = options.string + else: - if options.verbose: + if options.verbose: sys.stderr.write("Reading from standard input.\n") - + stdin_universal = os.fdopen(os.dup(sys.stdin.fileno()), "rU") rawtext = stdin_universal.read() - + if rawtext == "" or rawtext is None: print "No input. Pipe a file to convert to the script,\n" + \ "or call with -f. Call with --help for more info." @@ -208,16 +215,16 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", # rather than parsing the file to make sure. (Parsing will fail # below if the guess is wrong on OFX/1 and QIF.) filetype = ofx.FileTyper(rawtext).trust() - + if options.type: print "Input file type is %s." % filetype sys.exit(0) elif options.debug: sys.stderr.write("Input file type is %s.\n" % filetype) - - converted = convert(rawtext, filetype, verbose=options.verbose, - fid=options.fid, org=options.org, bankid=options.bankid, - accttype=options.accttype, acctid=options.acctid, + + converted = convert(rawtext, filetype, verbose=options.verbose, + fid=options.fid, org=options.org, bankid=options.bankid, + accttype=options.accttype, acctid=options.acctid, balance=options.balance, curdef=options.curdef, lang=options.lang, dayfirst=options.dayfirst, debug=options.debug) From a877d7f9251c431cd47a14f78e7b575c507f42ad Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 2 Dec 2010 15:11:39 -0200 Subject: [PATCH 04/31] ignora arquivos .swp --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0d20b64..c9b568f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +*.swp From c845d00a8aca3d02f33b6da0ba0d05cbba45c701 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Fri, 3 Dec 2010 11:06:58 -0200 Subject: [PATCH 05/31] remove \r from raw_string --- fixofx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fixofx.py b/fixofx.py index 4af1cf7..d5f5e8f 100755 --- a/fixofx.py +++ b/fixofx.py @@ -175,6 +175,7 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", try: srcfile = open(options.filename, 'rU') + import ipdb; ipdb.set_trace() rawtext = srcfile.read() srcfile.close() except StandardError, detail: @@ -191,7 +192,7 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", elif options.string: if options.verbose: sys.stderr.write("Reading from string\n") - rawtext = options.string + rawtext = options.string.replace('\r','') else: if options.verbose: From 5579eff09b6f28188d23481ad1780a219db2a7ad Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 8 Dec 2010 14:52:04 -0200 Subject: [PATCH 06/31] fix bad ofc by removing inline closing "tags" --- lib/ofxtools/ofc_parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 46e7673..ba1049c 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -17,7 +17,7 @@ # # ofxtools.ofc_parser - parser class for reading OFC documents. # - +import re import ofxtools from pyparsing import alphanums, CharsNotIn, Dict, Forward, Group, \ Literal, OneOrMore, White, Word, ZeroOrMore @@ -52,6 +52,12 @@ def _tag(self, closed=True): def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" + ofc = self.fix_malformed_ofc(ofc) return self.parser.parseString(ofc).asDict() + def fix_malformed_ofc(self, ofc): + """ + Fix an OFC, by removing inline closing 'tags' + """ + return re.compile(r'(\w+.*)<\/\w+>', re.UNICODE).sub(r'\1', ofc) From af5ff9fec91a0b8309b4662a595d38352a3fe8a8 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 8 Dec 2010 17:04:35 -0200 Subject: [PATCH 07/31] enables !Type:Invst --- lib/ofxtools/qif_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ofxtools/qif_parser.py b/lib/ofxtools/qif_parser.py index 3720269..62027e6 100644 --- a/lib/ofxtools/qif_parser.py +++ b/lib/ofxtools/qif_parser.py @@ -112,7 +112,7 @@ def __init__(self, debug=False): self.parser = Group(ZeroOrMore(White()).suppress() + ZeroOrMore(acctlist).suppress() + - OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns) + + OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns | invsttxns) + ZeroOrMore(White()).suppress() ).setResultsName("QifStatement") From 3eaef4bc1f1873f98a5eef502b35b380faff6075 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 8 Dec 2010 17:06:15 -0200 Subject: [PATCH 08/31] removes silly ipdb --- fixofx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fixofx.py b/fixofx.py index d5f5e8f..d5aff30 100755 --- a/fixofx.py +++ b/fixofx.py @@ -175,7 +175,6 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", try: srcfile = open(options.filename, 'rU') - import ipdb; ipdb.set_trace() rawtext = srcfile.read() srcfile.close() except StandardError, detail: From bd093a5338314016e0a00165cda75146dc80787e Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 9 Dec 2010 12:02:37 -0200 Subject: [PATCH 09/31] fixes bad ofc and tries to parse --- lib/ofxtools/ofc_parser.py | 35 ++++- test/fixtures/bad.ofc | 290 +++++++++++++++++++++++++++++++++++++ test/test_suite.py | 19 +-- 3 files changed, 332 insertions(+), 12 deletions(-) create mode 100644 test/fixtures/bad.ofc diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index ba1049c..5c95f8c 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -21,6 +21,7 @@ import ofxtools from pyparsing import alphanums, CharsNotIn, Dict, Forward, Group, \ Literal, OneOrMore, White, Word, ZeroOrMore +from pyparsing import ParseException class OfcParser: """Dirt-simple OFC parser for interpreting OFC documents.""" @@ -52,12 +53,40 @@ def _tag(self, closed=True): def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" - ofc = self.fix_malformed_ofc(ofc) - return self.parser.parseString(ofc).asDict() + ofc = self.remove_inline_closing_tags(ofc) + try: + return self.parser.parseString(ofc).asDict() + except ParseException: + fixed_ofc = self.fix_ofc(ofc) + return self.parser.parseString(fixed_ofc).asDict() - def fix_malformed_ofc(self, ofc): + def remove_inline_closing_tags(self, ofc): """ Fix an OFC, by removing inline closing 'tags' """ return re.compile(r'(\w+.*)<\/\w+>', re.UNICODE).sub(r'\1', ofc) + def fix_ofc(self, ofc): + """ + Do some magic to fix an bad OFC + """ + ofc = self._remove_bad_tags(ofc) + ofc = self._fill_dummy_tags(ofc) + return self._inject_tags(ofc) + + def _remove_bad_tags(self, ofc): + ofc_without_trnrs = re.sub(r'<[/]*TRNRS>', '', ofc) + return re.sub(r'<[/]*CLTID>\w+', '', ofc_without_trnrs) + + def _fill_dummy_tags(self, ofc): + expression = r'(<%s>)[^\w+]' + replacement = r'<%s>0\n' + ofc = re.sub(expression % 'FITID', replacement % 'FITID' , ofc) + filled_ofc = re.sub(expression % 'CHKNUM', replacement % 'CHKNUM' , ofc) + + return filled_ofc + + def _inject_tags(self, ofc): + tags ="\n\n\n0\n0\n0\n\n" + if not re.findall(r'\w*\s*', ofc): + return ofc.replace('', tags).replace('', '\n') diff --git a/test/fixtures/bad.ofc b/test/fixtures/bad.ofc new file mode 100644 index 0000000..9369871 --- /dev/null +++ b/test/fixtures/bad.ofc @@ -0,0 +1,290 @@ + + +0310166442 + +20080710 +20080811 +1061.03 + + +1 +20080710 +742.50 + + +DOC 409.0190INFNET EDUCA + + + + +1 +20080711 +-500.00 + + +CEI SAQUE 000042.001073 + + + + +1 +20080711 +-150.00 + + +CEI SAQUE 000059.001073 + + + + +1 +20080711 +1900.97 + + +TBI 6012.01284-2EBBS + + + + +1 +20080711 +-19.80 + + +TAR MAXCTA PJ MENS 06/08 + + + + +1 +20080714 +-17.16 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080714 +-40.00 + + +TBI 0301.75492-5EBBS + + + + +1 +20080714 +-191.00 + + +TBI 0726.41321-4Eduardo + + + + +1 +20080714 +-1720.22 + + +DOC BKI 089570 INFORMAL + + + + +1 +20080714 +-7.80 + + +TAR DOC BKI + + + + +1 +20080715 +-2.99 + + +RSHOP-DROG DESCON-001073 + + + + +1 +20080717 +105.54 + + +TEC DEP CHEQUE + + + + +1 +20080722 +-70.00 + + +CEI SAQUE 000349.001073 + + + + +1 +20080724 +135.48 + + +TBI 0769.31862-7 C/C + + + + +1 +20080728 +-9.58 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080728 +-14.68 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080728 +-15.95 + + +RSHOP-MOLL ROSARI-001073 + + + + +1 +20080729 +-80.00 + + +CEI SAQUE 000661.001073 + + + + +1 +20080729 +450.00 + + +TEC DEPOSITO DINHEIRO + + + + +1 +20080730 +-50.00 + + +CEI SAQUE 000182.001073 + + + + +1 +20080730 +-10.00 + + +RSHOP-RESTAURANTE-001073 + + + + +1 +20080731 +-25.00 + + +RSHOP-RIO NORTE -001073 + + + + +1 +20080731 +-308.85 + + +BKI TELEMAR OI 125168480 + + + + +1 +20080801 +-40.00 + + +CEI SAQUE 001925.001073 + + + + +1 +20080804 +-19.80 + + +TAR MAXCTA PJ MENS 07/08 + + + + +1 +20080805 +-30.00 + + +RSHOP-NOVA PARTNE-001073 + + + + +1 +20080811 +-19.36 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080811 +1059.48 + + +TBI 6012.01284-2EBBS + + + + + diff --git a/test/test_suite.py b/test/test_suite.py index f9b583d..c2a84b7 100755 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,15 +25,16 @@ import unittest def suite(): - modules_to_test = ['ofxtools_qif_converter', 'mock_ofx_server', - 'ofx_account', 'ofx_builder', 'ofx_client', - 'ofx_document', 'ofx_error', 'ofx_parser', - 'ofx_request', 'ofx_response', 'ofx_validators'] + modules_to_test = ['ofxtools_qif_converter', 'mock_ofx_server', + 'ofx_account', 'ofx_builder', 'ofx_client', + 'ofx_document', 'ofx_error', 'ofx_parser', + 'ofx_request', 'ofx_response', 'ofx_validators', + 'ofc_parser'] alltests = unittest.TestSuite() - + for module in map(__import__, modules_to_test): alltests.addTest(unittest.findTestCases(module)) - + return alltests if __name__ == '__main__': From c3b82649f1f59082b54461e34ecce01f0896bcf3 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 9 Dec 2010 12:02:37 -0200 Subject: [PATCH 10/31] fixes bad ofc and tries to parse --- lib/ofxtools/ofc_parser.py | 35 ++++- test/fixtures/bad.ofc | 290 +++++++++++++++++++++++++++++++++++++ test/ofc_parser.py | 31 ++++ test/test_suite.py | 19 +-- 4 files changed, 363 insertions(+), 12 deletions(-) create mode 100644 test/fixtures/bad.ofc create mode 100644 test/ofc_parser.py diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index ba1049c..5c95f8c 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -21,6 +21,7 @@ import ofxtools from pyparsing import alphanums, CharsNotIn, Dict, Forward, Group, \ Literal, OneOrMore, White, Word, ZeroOrMore +from pyparsing import ParseException class OfcParser: """Dirt-simple OFC parser for interpreting OFC documents.""" @@ -52,12 +53,40 @@ def _tag(self, closed=True): def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" - ofc = self.fix_malformed_ofc(ofc) - return self.parser.parseString(ofc).asDict() + ofc = self.remove_inline_closing_tags(ofc) + try: + return self.parser.parseString(ofc).asDict() + except ParseException: + fixed_ofc = self.fix_ofc(ofc) + return self.parser.parseString(fixed_ofc).asDict() - def fix_malformed_ofc(self, ofc): + def remove_inline_closing_tags(self, ofc): """ Fix an OFC, by removing inline closing 'tags' """ return re.compile(r'(\w+.*)<\/\w+>', re.UNICODE).sub(r'\1', ofc) + def fix_ofc(self, ofc): + """ + Do some magic to fix an bad OFC + """ + ofc = self._remove_bad_tags(ofc) + ofc = self._fill_dummy_tags(ofc) + return self._inject_tags(ofc) + + def _remove_bad_tags(self, ofc): + ofc_without_trnrs = re.sub(r'<[/]*TRNRS>', '', ofc) + return re.sub(r'<[/]*CLTID>\w+', '', ofc_without_trnrs) + + def _fill_dummy_tags(self, ofc): + expression = r'(<%s>)[^\w+]' + replacement = r'<%s>0\n' + ofc = re.sub(expression % 'FITID', replacement % 'FITID' , ofc) + filled_ofc = re.sub(expression % 'CHKNUM', replacement % 'CHKNUM' , ofc) + + return filled_ofc + + def _inject_tags(self, ofc): + tags ="\n\n\n0\n0\n0\n\n" + if not re.findall(r'\w*\s*', ofc): + return ofc.replace('', tags).replace('', '\n') diff --git a/test/fixtures/bad.ofc b/test/fixtures/bad.ofc new file mode 100644 index 0000000..9369871 --- /dev/null +++ b/test/fixtures/bad.ofc @@ -0,0 +1,290 @@ + + +0310166442 + +20080710 +20080811 +1061.03 + + +1 +20080710 +742.50 + + +DOC 409.0190INFNET EDUCA + + + + +1 +20080711 +-500.00 + + +CEI SAQUE 000042.001073 + + + + +1 +20080711 +-150.00 + + +CEI SAQUE 000059.001073 + + + + +1 +20080711 +1900.97 + + +TBI 6012.01284-2EBBS + + + + +1 +20080711 +-19.80 + + +TAR MAXCTA PJ MENS 06/08 + + + + +1 +20080714 +-17.16 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080714 +-40.00 + + +TBI 0301.75492-5EBBS + + + + +1 +20080714 +-191.00 + + +TBI 0726.41321-4Eduardo + + + + +1 +20080714 +-1720.22 + + +DOC BKI 089570 INFORMAL + + + + +1 +20080714 +-7.80 + + +TAR DOC BKI + + + + +1 +20080715 +-2.99 + + +RSHOP-DROG DESCON-001073 + + + + +1 +20080717 +105.54 + + +TEC DEP CHEQUE + + + + +1 +20080722 +-70.00 + + +CEI SAQUE 000349.001073 + + + + +1 +20080724 +135.48 + + +TBI 0769.31862-7 C/C + + + + +1 +20080728 +-9.58 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080728 +-14.68 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080728 +-15.95 + + +RSHOP-MOLL ROSARI-001073 + + + + +1 +20080729 +-80.00 + + +CEI SAQUE 000661.001073 + + + + +1 +20080729 +450.00 + + +TEC DEPOSITO DINHEIRO + + + + +1 +20080730 +-50.00 + + +CEI SAQUE 000182.001073 + + + + +1 +20080730 +-10.00 + + +RSHOP-RESTAURANTE-001073 + + + + +1 +20080731 +-25.00 + + +RSHOP-RIO NORTE -001073 + + + + +1 +20080731 +-308.85 + + +BKI TELEMAR OI 125168480 + + + + +1 +20080801 +-40.00 + + +CEI SAQUE 001925.001073 + + + + +1 +20080804 +-19.80 + + +TAR MAXCTA PJ MENS 07/08 + + + + +1 +20080805 +-30.00 + + +RSHOP-NOVA PARTNE-001073 + + + + +1 +20080811 +-19.36 + + +RSHOP-MINI MAX TI-001073 + + + + +1 +20080811 +1059.48 + + +TBI 6012.01284-2EBBS + + + + + diff --git a/test/ofc_parser.py b/test/ofc_parser.py new file mode 100644 index 0000000..2e67a8d --- /dev/null +++ b/test/ofc_parser.py @@ -0,0 +1,31 @@ +#coding: utf-8 +import sys +sys.path.insert(0, '../3rdparty') +sys.path.insert(0, '../lib') + +from ofxtools.ofc_parser import OfcParser +from os.path import join, realpath, dirname +from pyparsing import ParseException + +import unittest + + +bad_ofc_path = join(realpath(dirname(__file__)), 'fixtures', 'bad.ofc') + +def assert_not_raises(function, param, exception): + try: + function(param) + except exception: + raise AssertionError, "Exception %s raised" %exception + + +class OFCParserTestCase(unittest.TestCase): + def setUp(self): + self.ofc = open(bad_ofc_path, 'r').read() + self.parser = OfcParser() + + def test_parsing_bad_ofc_should_not_raise_exception(self): + assert_not_raises(self.parser.parse, self.ofc, ParseException) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_suite.py b/test/test_suite.py index f9b583d..c2a84b7 100755 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,15 +25,16 @@ import unittest def suite(): - modules_to_test = ['ofxtools_qif_converter', 'mock_ofx_server', - 'ofx_account', 'ofx_builder', 'ofx_client', - 'ofx_document', 'ofx_error', 'ofx_parser', - 'ofx_request', 'ofx_response', 'ofx_validators'] + modules_to_test = ['ofxtools_qif_converter', 'mock_ofx_server', + 'ofx_account', 'ofx_builder', 'ofx_client', + 'ofx_document', 'ofx_error', 'ofx_parser', + 'ofx_request', 'ofx_response', 'ofx_validators', + 'ofc_parser'] alltests = unittest.TestSuite() - + for module in map(__import__, modules_to_test): alltests.addTest(unittest.findTestCases(module)) - + return alltests if __name__ == '__main__': From bc7322c8f0bd492db0a7d1527c5d9f03726c278f Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Fri, 14 Jan 2011 11:57:56 -0200 Subject: [PATCH 11/31] Handles special case where there is no bankinfo and TRNRS instead of ATTSTMT --- lib/ofxtools/.qif_parser.py.swp | Bin 16384 -> 0 bytes lib/ofxtools/ofc_converter.py | 26 ++++++++++++++++---------- test/ofc_parser.py | 10 ++++++++-- test/test_suite.py | 2 +- 4 files changed, 25 insertions(+), 13 deletions(-) delete mode 100644 lib/ofxtools/.qif_parser.py.swp diff --git a/lib/ofxtools/.qif_parser.py.swp b/lib/ofxtools/.qif_parser.py.swp deleted file mode 100644 index 37f15c01701ecb695eaa50330a2f7abbdecac716..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOUu+{s86Qf4{^9s{5<(!Zb8aQuoMWd!2%7dD94B{}CiW#x8m`S`2F*Quf(F!P1`N?FiQ}S*f_oN^QRHwD+Tf#hWEuO2@H8=^AfuyM`y2 zS9JSR>P>H_45SPk!GJH@GoL#}K0WvFtSb3n<}5w^k&PpS(#I(SDFZ13DFZ13DFZ13 zDFZ13DFgpc3`qAb@+CCruCP(jj^JJ4b##1pc>Tzfa^datkTQ@mkTQ@mkTQ@mkTQ@m zkTQ@mkTQ@mkTQ@ma0fDAv6P1TmeeJf8Gb%fE$1XJO2 zUIU&67Jz$!dw^T-A>;?ZHQ*6|06)K*kZ%BA2N+NSiooxY6!{(SZJ-8x9QZ8~DYt-c z0$%_=3j7A#YJQXSKYb^DWO^m}qF{233Ez^UX7pGtQ!zw$-7{=qn36lT$m9mF!FZYySR=c>(CF_Zt?bzJjWkU9tErWac!a%(V%jCW-C_PUv zdXDerU@Nnj5X%tv3@evCyWV%%ylO%gE%RO1!_!=zo`cIx{Y`SPVAwl%CI?MemWTt@ zgsB{a{RBp)Gg;StIiCf26fDYmdzsrT&gx;D24rU!vvi(j7MU6qH7%vwF;hsOWK9=eH})H5m$iJ0#gz3cPzQ}sBvq|g1S}`U6ZU2TV{qf$9caU(jN-! zQWHg+c_D$Z&YhqxypdfV6*SGH)Yr9O#p(695SW>gFtMU#R#?wTQuP$WS2Z{s*y9T3!?kk%UVH$nZB1$K`=7QdS!@d7@xG2 z2JKX#C&K4Ynna4H9kVmqpc?BKlJByHYw>7kCh@GmCF4>FI0y!`7E0bBl_X78XEKtX0byQi$ zLC2yv)d7lO;mj+~MG8DbOv__M9Gd`|1;9Zg(i}xsE64SQuIeqSFFZ0n?HV zJHq%>)ewqAUo_*>OY$J4xGWA@LT4zV1ca_#X1A#9GTOt8ffGYgF6e6YLJ?9Bvs5=? zszj7def4p+F1(D{AHfOA&<11iTeR&2MTE;t-sUESYj=F3!)V9ZWgdd>4t1FqF0g{) zi*MAjSzZ5pePf+&me<4qrd%%sJtaq$9<4FhPV+@RG)hAxyF)kXnzY}hK~M+aT=IlP%R z`_you!8Do{LyPv5PNU;7Jd+OPHr>PWhNcy$aN2SY%XL6HJfbV5J6|g8?d=r}e1G0$ zMaS!uP?us*l$NWNrCMWYX0AA^MJ_u&?HPUQG^K%h4fKI>Bi zXk44aK(=o(7aeKqPPVw3T<9K1i&vt!3i~lmR9V{?Jt@#9@I5|ypTa(xQ;w1eg);d^ zwZRqG=kZQg(z)5$&&>=YTw*_Vj-PUIx0r3{U|6 zggyNC0Snjxioidxe}4hk0v-U)0QUhmv4?j73OoRu0se~p`y0SZz%#&;z^8z>uz$Y^ zd=2<2-~x{UYH$BXwC@jUFAv-RRC_N1=`Up08D*qQ+sR|aTu6NSdK*>bLPM0275Lu#+Ntzc3 z=#Vg%y}GoqMz5;e*=S}oTk$=Q*=Ap*#0I*;)&}{=K{Tl{o2p6M76P7pyu4C1c!Hn< ziKLb;I+`+7MGzPdlcSoykd}dR70<;VsHj3oiX#3Z=utUU^tjTge(Hv{2sJ--g{WHt z@VVC0NgJmD`H1oRDr=S(HDkI14AH!3;3 zfyw0$8;v7+hdyVThS$=`;p^0la^=_QAZ2_V8wR7vk+$foQsZ7ZQY$WEHeuH$wI`3} z)FX))*YK=hD8!@>Xwh*K@)eaTUdC8Dl*gFP;lL5O(UfQG-Gtsjs&);m9-p+L7Ya)K z)NFc0!|*ui3sDy5oo_>!&pj-aAHEckCu%`C*ts940pqWCMmGP^923@CwcAc6AAfTc zIo)9`3x!YAu2!>7A6_2p9cL@5TSw@)5}(G0D?sFBY6u=&*ZgE% G1@d17-_G>_ diff --git a/lib/ofxtools/ofc_converter.py b/lib/ofxtools/ofc_converter.py index 52ce57a..be414b9 100644 --- a/lib/ofxtools/ofc_converter.py +++ b/lib/ofxtools/ofc_converter.py @@ -73,16 +73,22 @@ def __init__(self, ofc, fid="UNKNOWN", org="UNKNOWN", curdef=None, if self.debug: sys.stderr.write("Extracting document properties.\n") - try: - self.bankid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["BANKID"] - acct_code = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCTTYPE"] - self.accttype = self.acct_types.get(acct_code, "UNKNOWN") - self.acctid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCTID"] - except KeyError: - self.bankid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["BANKID"] - acct_code = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["ACCTTYPE"] - self.accttype = self.acct_types.get(acct_code, "UNKNOWN") - self.acctid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["ACCTID"] + if self.parsed_ofc["document"]["OFC"].asDict().has_key('TRNRS'): + #TRNRS has almost the same info of ACCTSTMT. Just with another name. Damn you Banks! + self.parsed_ofc["document"]["OFC"]['ACCTSTMT'] = self.parsed_ofc["document"]["OFC"]["TRNRS"] + + if self.parsed_ofc["document"]["OFC"]["ACCTSTMT"].asDict().has_key('ACCTFROM'): + # Bank info ignored if not exists + try: + self.bankid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["BANKID"] + acct_code = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCTTYPE"] + self.accttype = self.acct_types.get(acct_code, "UNKNOWN") + self.acctid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCTID"] + except KeyError: + self.bankid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["BANKID"] + acct_code = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["ACCTTYPE"] + self.accttype = self.acct_types.get(acct_code, "UNKNOWN") + self.acctid = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["ACCTFROM"]["ACCOUNT"]["ACCTID"] self.balance = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["STMTRS"]["LEDGER"] self.start_date = self.parsed_ofc["document"]["OFC"]["ACCTSTMT"]["STMTRS"]["DTSTART"] diff --git a/test/ofc_parser.py b/test/ofc_parser.py index 2e67a8d..bac198b 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -9,8 +9,9 @@ import unittest - -bad_ofc_path = join(realpath(dirname(__file__)), 'fixtures', 'bad.ofc') +FIXTURES_PATH = join(realpath(dirname(__file__)), 'fixtures') +bad_ofc_path = join(FIXTURES_PATH, 'bad.ofc') +no_bankinfo_ofc_path = join(FIXTURES_PATH, 'nobankinfo_and_trnrs.ofc') def assert_not_raises(function, param, exception): try: @@ -27,5 +28,10 @@ def setUp(self): def test_parsing_bad_ofc_should_not_raise_exception(self): assert_not_raises(self.parser.parse, self.ofc, ParseException) + def test_parsing_ofc_without_bank_info_not_raise_Exception(self): + self.ofc = open(no_bankinfo_ofc_path, 'r').read() + assert_not_raises(self.parser.parse, self.ofc, Exception) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_suite.py b/test/test_suite.py index c2a84b7..9c7e450 100755 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -29,7 +29,7 @@ def suite(): 'ofx_account', 'ofx_builder', 'ofx_client', 'ofx_document', 'ofx_error', 'ofx_parser', 'ofx_request', 'ofx_response', 'ofx_validators', - 'ofc_parser'] + 'ofc_parser', 'ofc_converter'] alltests = unittest.TestSuite() for module in map(__import__, modules_to_test): From b6d7b510b5ae19b098616f96952da6bb3b7f3062 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Fri, 25 Mar 2011 16:46:11 -0300 Subject: [PATCH 12/31] ignoring stuff --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c9b568f..1559f2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc *.swp +*.swo From 55d7e8e3951971a7db686ac0fbacfd182465f37c Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Fri, 25 Mar 2011 16:48:52 -0300 Subject: [PATCH 13/31] tests for the ofcconverter --- test/fixtures/nobankinfo_and_trnrs.ofc | 172 +++++++++++++++++++++++++ test/ofc_converter.py | 47 +++++++ 2 files changed, 219 insertions(+) create mode 100644 test/fixtures/nobankinfo_and_trnrs.ofc create mode 100644 test/ofc_converter.py diff --git a/test/fixtures/nobankinfo_and_trnrs.ofc b/test/fixtures/nobankinfo_and_trnrs.ofc new file mode 100644 index 0000000..65202a4 --- /dev/null +++ b/test/fixtures/nobankinfo_and_trnrs.ofc @@ -0,0 +1,172 @@ + +2 +1252 + +0 +20110113152823 +1246578033 +0 +0 +0 + + +0 +4 + +********************************************** +BRADESCO - Mensagens Informativas +********************************************** +Obrigado por Utilizar o Nosso Sistema +______________________________________________ + + + +1 +0 + +20101214 +20110113 +350.66 + + +0 +20110110 +24.98 +10012011 3176230 00000 +3176230 +00112 TRANSF.AUT. C/C - Juliano Alberto Gimenes + + + + +0 +20110110 +5.00 +10012011 8859143 00000 +8859143 +00351 DEP CC AUTOAT - Ag02188maq018859seq02143 + + + + +1 +20110110 +-13.91 +10012011 0004555 00000 +0004555 +00901 VISA ELECTRON - CACULA + + + + +1 +20110110 +-5.80 +10012011 0163279 00000 +0163279 +00901 VISA ELECTRON - ALTERO RJ + + + + +1 +20110110 +-22.80 +10012011 0683403 00000 +0683403 +00901 VISA ELECTRON - CASA DO CABELEIREIRO + + + + +1 +20110110 +-51.80 +10012011 0781332 00000 +0781332 +00901 VISA ELECTRON - PALACIO DOS CRISTAIS + + + + +0 +20110111 +100.00 +11012011 3826792 00000 +3826792 +00351 DEP CC AUTOAT - Ag00553maq023826seq04792 + + + + +1 +20110111 +-134.10 +11012011 0381590 00000 +0381590 +00901 VISA ELECTRON - VIA MILANO + + + + +1 +20110111 +-10.60 +11012011 0750139 00000 +0750139 +00901 VISA ELECTRON - VIA NIA + + + + +1 +20110111 +-103.00 +11012011 0809630 00000 +0809630 +00901 VISA ELECTRON - SONHO DOS PES + + + + +0 +20110112 +63.00 +12012011 0244443 00000 +0244443 +00412 TRANSF AUTORIZ - Masayuki Missao + + + + +1 +20110112 +-19.90 +12012011 0120055 00000 +0120055 +00901 VISA ELECTRON - ANTONELLA + + + + +1 +20110112 +-159.00 +12012011 0563632 00000 +0563632 +00901 VISA ELECTRON - ANDARELLA + + + + +1 +20110112 +-29.90 +12012011 0781387 00000 +0781387 +00901 VISA ELECTRON - PALACIO DOS CRISTAIS + + + + + \ No newline at end of file diff --git a/test/ofc_converter.py b/test/ofc_converter.py new file mode 100644 index 0000000..aa509fe --- /dev/null +++ b/test/ofc_converter.py @@ -0,0 +1,47 @@ +#coding: utf-8 +import sys +sys.path.insert(0, '../3rdparty') +sys.path.insert(0, '../lib') + +from ofxtools.ofc_converter import OfcConverter +from os.path import join, realpath, dirname + +import unittest + + +no_bankinfo_ofc_path = join(realpath(dirname(__file__)), 'fixtures', 'nobankinfo_and_trnrs.ofc') + +def assert_not_raises(function, param, exception): + try: + function(param) + except exception: + raise AssertionError, "Exception %s raised" %exception + + +class OFCConverterWithNoBankInfoTestCase(unittest.TestCase): + """ + Testing an special case that doesn't has the bank info 'tags' + and instead of having ACCTSTMT, it has TRNRS which has the same + information needed + """ + + def setUp(self): + self.ofc = open(no_bankinfo_ofc_path, 'r').read() + + def test_converting_ofc_with_no_bankinfo_should_not_raise_KeyError(self): + assert_not_raises(OfcConverter, self.ofc, KeyError) + + def test_ofc_converter_getting_balance_value(self): + ofc_converter = OfcConverter(self.ofc) + self.assertEqual(ofc_converter.balance, '350.66') + + def test_ofc_converter_getting_start_date(self): + ofc_converter = OfcConverter(self.ofc) + self.assertEqual(ofc_converter.start_date, '20101214') + + def test_ofc_converter_getting_end_date(self): + ofc_converter = OfcConverter(self.ofc) + self.assertEqual(ofc_converter.end_date, '20110113') + +if __name__ == '__main__': + unittest.main() From f18338e9df697d38793e47e02843bbfa1221bd74 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Fri, 25 Mar 2011 16:50:17 -0300 Subject: [PATCH 14/31] translates chknum to checknum --- lib/ofxtools/ofc_parser.py | 10 +++- test/fixtures/ofc_with_chknum.ofc | 81 +++++++++++++++++++++++++++++++ test/ofc_parser.py | 8 +++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/ofc_with_chknum.ofc diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 5c95f8c..6f279c4 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -54,6 +54,7 @@ def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" ofc = self.remove_inline_closing_tags(ofc) + ofc = self._translate_chknum_to_checknum(ofc) try: return self.parser.parseString(ofc).asDict() except ParseException: @@ -82,10 +83,17 @@ def _fill_dummy_tags(self, ofc): expression = r'(<%s>)[^\w+]' replacement = r'<%s>0\n' ofc = re.sub(expression % 'FITID', replacement % 'FITID' , ofc) - filled_ofc = re.sub(expression % 'CHKNUM', replacement % 'CHKNUM' , ofc) + filled_ofc = re.sub(expression % 'CHECKNUM', replacement % 'CHECKNUM' , ofc) return filled_ofc + def _translate_chknum_to_checknum(self, ofc): + """ + Some banks put an CHKNUM instead of CHECKNUM. this method translates + CHKNUM to CHECKNUM in order to parse this information correctly + """ + return re.sub('CHKNUM', 'CHECKNUM', ofc) + def _inject_tags(self, ofc): tags ="\n\n\n0\n0\n0\n\n" if not re.findall(r'\w*\s*', ofc): diff --git a/test/fixtures/ofc_with_chknum.ofc b/test/fixtures/ofc_with_chknum.ofc new file mode 100644 index 0000000..185b2a1 --- /dev/null +++ b/test/fixtures/ofc_with_chknum.ofc @@ -0,0 +1,81 @@ + + 2 + 1252 + + + 001 + 3071-6 + 00000071692 + 0 + + + 20080630 + 20080731 + 480.58 + + 1 + 20080711 + -60.57 + 20080711160570 + 00049434 + Pagamento de Telefone + + + 1 + 20080714 + -147.36 + 200807141147360 + 32942835 + Pagto cartão crédito + + + 1 + 20080714 + -25.78 + 20080714125780 + 11058928 + Cobrança de Juros + + + 0 + 20080715 + 1000.00 + 2008071501000000 + 00972676 + DOC Crédito em Conta - 745 0094 7894868736 ALLAN ALBAREZ + + + 1 + 20080721 + -55.00 + 20080721155000 + 00028718 + Mesada + + + 1 + 20080724 + -9.00 + 2008072419000 + 00080724 + Tarifa Pacote de Serviços - Tarifa referente a 24/07/2008 + + + 1 + 20080725 + -2.50 + 2008072512500 + 00080602 + Tarifa SMS - Mês Anterior - Tarifa referente a 02/06/2008 + + + 1 + 20080731 + -1.19 + 2008073111190 + 91100701 + Cobrança de I.O.F. + + + + \ No newline at end of file diff --git a/test/ofc_parser.py b/test/ofc_parser.py index bac198b..c5abc8d 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -12,6 +12,7 @@ FIXTURES_PATH = join(realpath(dirname(__file__)), 'fixtures') bad_ofc_path = join(FIXTURES_PATH, 'bad.ofc') no_bankinfo_ofc_path = join(FIXTURES_PATH, 'nobankinfo_and_trnrs.ofc') +ofc_with_chknum_path = join(FIXTURES_PATH, 'ofc_with_chknum.ofc') def assert_not_raises(function, param, exception): try: @@ -32,6 +33,13 @@ def test_parsing_ofc_without_bank_info_not_raise_Exception(self): self.ofc = open(no_bankinfo_ofc_path, 'r').read() assert_not_raises(self.parser.parse, self.ofc, Exception) + def test_chknum_to_checknum_translation(self): + self.ofc = open(ofc_with_chknum_path, 'r').read() + #ensure that the CHECKNUM was translated + self.assertTrue('CHECKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) + self.assertFalse('CHKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) + + if __name__ == '__main__': unittest.main() From cca18a8def5508f4865e354256455a75bdc449e1 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 6 Jul 2011 16:15:37 -0300 Subject: [PATCH 15/31] removes swp --- lib/ofx/.client.py.swp | Bin 16384 -> 0 bytes lib/ofx/.document.py.swp | Bin 12288 -> 0 bytes lib/ofx/.parser.py.swp | Bin 20480 -> 0 bytes lib/ofxtools/.ofc_parser.py.swp | Bin 12288 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/ofx/.client.py.swp delete mode 100644 lib/ofx/.document.py.swp delete mode 100644 lib/ofx/.parser.py.swp delete mode 100644 lib/ofxtools/.ofc_parser.py.swp diff --git a/lib/ofx/.client.py.swp b/lib/ofx/.client.py.swp deleted file mode 100644 index 24b0e74604d9a99987ce492e5ea8c57c17ff5d9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHO-H#hr6(3r@AruiIkWi)5v8vrjUaxmcOOvt^j^k{tWWAB?-6Tp|>w4ze9(FwA z%-mVWLP{y35uhRg;tla3sF3(-UZMX$p?o~>Pzhe(2_Zxgkw83A_? z9gb^L9A{IRiNZIRiNZIRiNZIRiNZIRpO( z45-n)#?!Flp2U{($?sbZ{QfIBHk0dH2kyU?+}Cvk-23W*`^jnklQWPrkTZ}okTZ}o zkTZ}okTZ}okTZ}okTZ}o@Lym+*oJWl{XWkde!Txr_y4!fTw^bfGfZSpaOgXSO@<6UbF$a!2Q7Q-(wiR16~3? z3%I}sfxo}oF#ZO-4BP-JzyrXq-(?t|2R;XE0!zRv?=*}r0?z?WpaA^y9fole@PPY( zm)>p|KLG;ZDzFax^=;4_xB>KlW5AE!3Oj%;;MTo{@pa%d@G6QVKL=g}o&gHL%ixW< z`%YRgc@um{nL}Q)|k0{lGzz>RKOC=oll#ywq{|uyx0*^xJ_DY>ldUhv}S0fp! zWno!#KSXD>lau#eAt;6xoTg7$^$*c2+EVk(ddko*%?2My^X*HKI0D)?Gl@A#89=$HC2M0qsi$8-?$h$v1~PsRyG3=iODG9G7+W}l^P95^_# zEI7Isxh2yyr&6+>fxWTc5C}YIruRsMU_&!t`&OjsaC%c2kk`VTFm03en!2U$>`PA% z-X)j|BPf|Qv;h1IOM$Me5DP5;`po8k;wd%QgV{?K0R%I7AS|&X))o+oV@j&2l zfGIe2$&su*tc+H=T4|oiD4iiBbx(=;<)wq~v0s*B$E5Kw)8lo8#_i|k*STcfVLncX z={R1`U(SeK7`9X1(_yk`0H;RJNsjHloBYhM_)pm0b2_8I?>Vk~V=U)rJaD9o{e~$# z-S7V&IXeC{6)UGAN3j&Wobh71Rw}Ra48eb|m&$y7D3ro_Fk4zI?P zK60`kvJ_9}lB)U>Sn|7{l$I(HHGHysujllmKycbd{)m4;;E#EE(>HJ}q&VSXJ>ms84&itSjLA&< zNYWC{2V-U8+=Fw&04pf3nOI77K7wTjNzMpDk;eFewr~IiTr9s#Dz{Ek6*epFA~lZ_ z58;JFhq;Gbp|l8a+>nZM%G8Ox;#vN17nA^?f9L9(O;IU^5Kaf9zpM zxma&MzP;0?i!4h3p z!lJQWlKVk2NF9&Hfp$A~Gk#BvG2XN<2**IF(W%wd@pxPk`2Kt#OMcK_g)C+;Ts>c} z)*7wa3WlzJn=ELDNEzWQPN);!387V>Nz&5u5e$e~HGpwE#U{f8X(@*p z)E1m6>KJ3sWSp1$nI)>Cbxa!-X%~q)k9(e4micirdB{BRm{zz7{Q{>yP_$kyKfJPD zUMtf@8Hz48V0F)eODvy*?iA?{f(_pP{|NiwFJphq`~UR){VUk#-vnL&ZUD~$8^Fhb zW56x!`ELSOfe~;L_z=K#fX@KW01co7`~iFXZvx){ZUD~$Tfn`*i`eIX7kC=D4m<+f z1N;{I{NDi20R!Ms;8pDNp9h`L1KTn2e_*VZycqylll4vKOK8yg|sawPps zWfrK)u4Y*fYqsJlsQFkd)m)1&+d>JxAB|R&zk)M~5+PN*2{Lc5KMM%0%6uSHx6kM( zSXs-&Kq2LxgVn@NJ0Vy<&zs|TTg!X)jHYcJ_1gM)%c$cm(R5`E)Q7$<3rETR7ByNe zk;}6)C6m5VH9K9`$2$%Yo^dRd)YJ$^0ZPPs;#qJND2)gd07T>}uHWN555YA&(h||h zMce0hD)Jh(WpFw-$QXm_1E!3)#BpJAA=#6oOO_fJA@Gn6Sqwa`JP5=P48#nA7ZX~N zu0rSytgb>vVNw;2)rbw_msZp1Dy>qmeM#{vHFCxe-_+Hb&YWm2J96kIU14A+#kXug zXMD_J_~3Gbw7GJ5#cb6#8#@=aTAQtAYj@3j@`_ohR@?Q)+3j|-(QNUBR=d*P!C!`c zIQi#5-X!q*ZGRL}~1m=BTPX%d^qqbutA(iPB`lJmo4Rm<8VIG(^oZd@@ cgVRTqH63uXbuDcMY4HoEc=A|Iub(jf4cMfpDF6Tf diff --git a/lib/ofx/.document.py.swp b/lib/ofx/.document.py.swp deleted file mode 100644 index 3b18b187f6059d8d790ea486091f2a24528da564..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO>87b74AUzClIkh0)@a!kBrCJneo^WBJ#$GCia+&aK@6y-py)bRyEx%VE+>)n;$0_xuV z+}l<4UcLI>tFL-@d)U6-dyzibTDNdLYFTf*{^suQKXk|X+7-(>5rHcstpe`5$5o7L zx>6q-6<#v-PBCzGrlMmV3P-NCPUPyyJ5{68{N{VZRep4p6HjGeTH*W>hgEaMK*hj= zF%X+k^XW&dr`FfnY*4eGper9Zd@$Q;RK-BWK*d1CK*d1CK*d1CK*d1C!2bmUCVa^H z0#@>tY&HGt{?NI5_E_C21}X+B1}X+B1}X+B1}X+B1}X+B1}X+B1}X;r2N@8qW&I3) z+5hNU5j_6?KmYsx-`;OozXW~`d>Qx@@Hp@o@R#>l)}MfH0-ph{0Ka&|vc3m=7x)hF zMc@wbc_0D?z&`Li@G$Vx_gdDEfFA-gU=8^Ddo1hwfDhDxKfc?tz78A#9|nH+F3Wlq zH~>Bf{N|k)4}1l91qgvBfxo`Pvc3U)7FYxR`F6{C6ZjSIC141A1b7Vi!`m$DC%|jK zw}Ec~Cx8Sxz!LE1hb`;Jz*m8N;Bnwz;Ohs#99RcF1~7;Jz(1t?7I+nifv12+fnS5; z*YbZ<`7hK7WmhCk90jREZnCm=N;~Lu>K9Gig=f0DN%oR{W=TJbpR2gN9N4r(bq}9= z6oW6fe}EzPoFXvy^KifZ`z+#q=I8p)JuWD-{$n-+qfO)m;|3XlA1A_(W%B2hVp77D zw9_VOaoUpF5uvQ}ALAf1G5IMGR{OOA}w#bo?~v=L*>rxqg(dg&P(=DOXyv!y1ufQ zmvH6jx9w-1Xf|nE7)fa|X(sdrEoo{tSx9V_5*OXs+KK=`e5o0PQ=km>RUj7x-({vP z6`aV(NS8*Dnx3CXMc&v8gnyUG&UKxgu!xFhS9eCE>%TaG5F+G?wOeZjYTu*;Ol?lEMlj z4kDn#L1(bl>k*W3`LN&H+S~3fb9Ils6B*!iK`_`gVrk7saEqO?eFHW=PN|9y*(Q?C7X+w^hPE}BZh1rq$^DCLF7#5M5 zA(%NSWF}u*RztACm}M8je=M_1jTLfK3WQZlqUh&PF4CbC$C|nUtL+5DL|j>#vb;3X z%E4NFSx!olDiRu{pC;4BWH*r{UJv%sIy-hyxyON;`O+Q3<+;2L(b&hKAfZYGHYcNjg614@Fr2dWRy&PnBFffC@i7iKE-XP) zsdY9jn=IB~gb#w!-D+pFl(stc_YS%nATX5;&I9Zlg!9RGPV9>a8#irCoHiB;!s1h$ zri2ShwUfMGFj`>g1RcYTChn7;nP^?hnX1kEpac{jdr##fkDjlkb&s|d9Ts_+s4%}S`x z+glilvA#F;OpTBgSfPdk zmLKTD)YGitOp^;QH+lyz>>VD^jn4jlr+?7v4rp(mw)Xnly#pK!c-o;(|0aF9*WYfC z#ET^&k*BP2nq>(L_#UJVu3WrJL>{yS1$rT3R*CWeY+&Tc?hL3mu<7~Epf_m1$6?^J z<4C>|dlCE$=OjYR;E16wv7#BzDaLsFP2>aBGK)NuNWE4gGlLYQOru67#)KQItFzgx zCGh?EM7C5kUWG4aP_ORxwz~a6w~0DQOm0Fka6FJ28o?R{oeA-j3QtZl^tQ&jk&I+{ zNDmh*j>QhdI5LE`1}sttn_n{1xA9V@n5)cRnW#<@W8TxC>*-;eY{@#C+nJHQaxDpX j3-O_HCx&p8f1_GiD7(9Pe diff --git a/lib/ofx/.parser.py.swp b/lib/ofx/.parser.py.swp deleted file mode 100644 index c5b0a69096497ba7df0678b2e9a5337fdf41afc7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeHOYm6jS6)t2^1Q8U1_#hs8U8j4er)PFtaWl>i>7563*zI1odl;6TWxQQ|r@MA) zs)~E7x2Itjh{i}_Gz1f4k2tW9z{_ue@#>A)*;v0XMsKJmJBYx*rb#=|m z4y*=)Kvnje>Z&^T_1$w%)w$U85&Cdx(!z0tWxeC!=MO(}&0W?5S6bF-;kBgF zzQ@mA&p|n(`pua

1^fe>Addi_1tc&HybX93q&*IN95@Ty z0$dCH9-Asp0FMLr0}H_4v9aHPIar&0IJiQ8sQjuvK?_1@`m z+&hSSg_)70+;CWaAieR_*usu54D+DK_PtPg!zmk%Laqm4fBPs1N76_<5*}q~kov+I zFifb|&hxlSLeDvNm@g!Sa2EqHWspR(()HFnzvrz8+xEL&xDxgQX-b}}K*Yu=lvxql zcAS?GY`OxHtx#xx^(918lg8cNx$zV)n*>>x4F@f1`F3|b%_`x*gdAmVNEkJN_6yZE zVn#V*Dg}i~Vo`*uYeP#Uv?NecqcRX`I>j?&+S#OB+=6+`Klq62J`s3hN(u9j04 z{~8>d1&h%tNEcy2!wzm@;%-oWH^}Fv$8$7FOTKF5g&TB)*In1Rkt^j2e462>w@T&L z)Dd6F{89&tIA16fpz{HbNT)QyS*5&bp@n5K=ogO2xyg1@XhhoYl2+rd(; z`eDU`rZ1~yFIO1$a+`c?&5S<8ny?_S(1t~}_osL+CeS4_Ri>tMl*aP~U-bkQ-FCX6 zqbas`ld(1`rOJsWZ<_XGS_f)zkjwX?qBU#H+Th?;Y*`0LDA_(k7U3Rc(cMhQr01l?(nPAYZuA&?IKD9ivQY_HdotQJi;G#vW0eNY$k3k9~DM@`Osg} zc)=fbG+yvHl1|jOnXYsfG@HAqA6_ffAFhr&l*}@v4#?41GLDkMr(G+b!^$kugc%jL zpOR004u)Pb-xr&UK-cyaH@`Rs<>FzQ?wNXbV28PZ_Hg&`E5D~z+|w%XtPIK6V%^hJ zB%?Ruj`hAs$>PJ$cs&)VzUBlCzewg?iWA@z)+EBv$!IhgtCz-Y(oczcL3O zLb`;KDC#=vPAJTKkGhr^{Nv$g76rBSI>akW6@>T$ZM zQk}ypfYX7VuvX)3OT2cmpdH!b4z-}#%7K@GMX;^M1cQ+lFn$K*%5|#Lb9AU&uhfgE zW6^WDVkK|yItukP`y>LiV2dW~hAVp9rhun`XYeN+k9fVC#ftCN5awQ@7X0I*6NbUm z_;|0^D+wf{cVx*|?QvAb6zcKAmD%}feSU1RG!d0K?ssWj^vQ3A0yPV~K?7W^ZlYO) z5bnp4o*%+J<9WdFRvmN^b^;$9+%el{YJ_J?sNsz zzz2XI!q4vmlfXs5F5nsX^!EaLf#=}Ie+zgFxECk^-$9)M;JVqb&yI|NjDd`SjDd`S zjDd`SjDd`S*C+#UE-c2-{f@VW<3Pp8ba{5R(Kvo|euiTZ@h+J3S~Dl6m%U|mvVhn{ zN@%fqQ?<5KU2dmEw}v-j_mN{n>PTI%_b0>+*O)iphNgKH~cYCrilZ?%j@gBva*vpFnn&=Al+c`>tuv)`J@S=E{#dY##7WXw+NyU23gU%@6vtuwV{TO?X2&T<7%Jtg9 znM38nUF*&_vg5ZJ}{fBg# zoVYDA9;1c)^;4UJnT6|FY|u(H3wsMA#{d5)%+ymC;1B%&L4Ns z1cGEFS>k|6wi+fHq{IEMn#6%*I^$S7KJl`IfmbkLz|6+?#{e<^WN`$tSnD^ z?n>XJ0wa&d;Bl zvyx{^XXw<)%MV!OrX2&00mp!2z%k$$a11yG90QJl{}TfyImSMOLyzQ6y_(;T9k}PO z?&cV93^)cH1C9a5fMdWh;23ZWI0hU8jseHOL&yN{;D7!{fyZDx{{J8U{{Qm?V}Ag5 zfFFQQfOX&n;CbNpXBhhp_zbuPoCJP6&e(0>OW+Hj2MC}8Yyl^LU!G>{XW%E`Q{Xyq z0l51VW8VWG16P1Ez&}qi_BZei@Hy}va2)vV3C6w!ZUF)CB5?O{#(o7p0q*SF`)j+i-OC^|Tn z>AQ#0gR=MC-=1@`$K9Gng>kQp<8Kw8>(wwY<;q0A=?luFI@FOtIuNn(OQq6e9?x%E zCB#W*dWfrDCPp~YM>!3gj1t01qycyl7*CYYP==i|T^-RK?Ninw>Uw0YS(v&*xN==G z5%Crut+K-+(qe(xr}4F4(4!!dS`?R9!SNt1(M2WGq+D98mS~2yrRtP2rw9axacDvrYwAixjrDiP@SdJT(?+0$kvsq+ z>zB4vDh>vs=+D9D@o7rGHGW##4Mbx1T@~%Lw;b5VX?`7CfAD>*WvZ{41_8(}X!BTc zmCc_~Vk=(@Q;^fh@zGh&f;PidQD&;khOlEzE<~`Pdy+T2*HfYgrV&kHHib?jqw9Pi z%B69|lq&l(d4?v7S{cul_x+sAP*#YISmfz8SwSCAAY+5tsZ?kh^6C8IVxG~Hp_a`l zyGKDj*D-M;f1irZ&`KYbU6-a7qd0cdPF$)?jXWg3gk}TgvlPmcXPu)qr%Mjo;~hl= zOqyysVm0R1r5bY7>0!zn>nmW}=0ldxEt~VLR-V&YIG09Njmi>*vKlHgtHVJO2^7W3 zs0*pAMncf6P};UjWpm?zj;bbzxYpF@23ZGGXjgf~dNhffM_Hf14SmHc^bgimazM$R zZLU^543Nsur1N+M`n@a}ys_KfD83$|nXek>H4a;{vxMfEr}^|qGtG`28(D2zfD*iO z*fXerjwl?2hL74sYR_hq{Ss`1frz#6A<#rWFAQaT*>|0YAnb+#MLh1MyeFt9ZwM9J zjS`^-q1IUs!q)*>wrg8&HZE_`_VVWDa(%0|)}+QJtu*SZwXIsCj<~a` z)&gSQPeTPg?LFcNb_jSI*%9%f-IJr1q)K=E&&9&Kie=b{cRi-rHdnDV2LoY{>CwS}bBbgf*xtB}E z(ol*ouj#|NZU_TK)@rR;fwaW%I1n@hX+ Date: Wed, 6 Jul 2011 16:31:03 -0300 Subject: [PATCH 16/31] removes duplication in test utils --- test/ofx_test_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/ofx_test_utils.py b/test/ofx_test_utils.py index 0274add..0101774 100644 --- a/test/ofx_test_utils.py +++ b/test/ofx_test_utils.py @@ -17,11 +17,15 @@ fixtures = os.path.join(os.path.dirname(__file__) or '.', "fixtures") def get_checking_stmt(): - return open(os.path.join(fixtures, "checking.ofx"), 'rU').read() + return _read_file("checking.ofx") def get_savings_stmt(): - return open(os.path.join(fixtures, "savings.ofx"), 'rU').read() + return _read_file("savings.ofx") def get_creditcard_stmt(): - return open(os.path.join(fixtures, "creditcard.ofx"), 'rU').read() + return _read_file("creditcard.ofx") + +def _read_file(filename): + return open(os.path.join(fixtures, filename), 'rU').read() + From 996b8523de53bae8dc2536e72ae69b291aea6bf7 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 7 Jul 2011 11:19:53 -0300 Subject: [PATCH 17/31] fixes problem with ofc with empty tags --- lib/ofxtools/ofc_parser.py | 6 + test/fixtures/evil_mailrs.ofx | 413 ++++++++++++++++++++++++++++++++++ test/ofc_parser.py | 8 + 3 files changed, 427 insertions(+) create mode 100644 test/fixtures/evil_mailrs.ofx diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 6f279c4..9454927 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -54,6 +54,7 @@ def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" ofc = self.remove_inline_closing_tags(ofc) + ofc = self.strip_empty_tags(ofc) ofc = self._translate_chknum_to_checknum(ofc) try: return self.parser.parseString(ofc).asDict() @@ -94,6 +95,11 @@ def _translate_chknum_to_checknum(self, ofc): """ return re.sub('CHKNUM', 'CHECKNUM', ofc) + def strip_empty_tags(self, ofc): + """Strips open/close tags that have no content.""" + strip_search = '<(?P[^>]+)>\s*' + return re.sub(strip_search, '', ofc) + def _inject_tags(self, ofc): tags ="\n\n\n0\n0\n0\n\n" if not re.findall(r'\w*\s*', ofc): diff --git a/test/fixtures/evil_mailrs.ofx b/test/fixtures/evil_mailrs.ofx new file mode 100644 index 0000000..0b59e85 --- /dev/null +++ b/test/fixtures/evil_mailrs.ofx @@ -0,0 +1,413 @@ + +2 +1252 + +0 +20110705182508 +1246577115 +0 +0 +0 + + +0 +4 + +********************************************** +BRADESCO - Mensagens Informativas +********************************************** +Obrigado por Utilizar o Nosso Sistema +______________________________________________ +********************************************** +JANDER SOUSA MARTINS +Agencia 3199 Conta 32306 +********************************************** +04/07 101 BX AUT APLIC 0040711 13893.58 + + + +0 +4 + + + + +1 +0 + +20110620 +20110705 +-13892.58 + + +0 +20110630 +25225.05 +30062011 7539185 00000 +7539185 +00062 RESG.DE PAPEIS + + + + +0 +20110630 +5.53 +30062011 0423385 00000 +0423385 +00282 DOC CRED.AUTOM* - EMBRATEL PARTICIPACOES SA + + + + +1 +20110630 +-10830.25 +30062011 0459648 00000 +0459648 +00412 TRANSF AUTORIZ - Marcos Rocha da Fonseca + + + + +1 +20110630 +-3000.00 +30062011 0172591 00000 +0172591 +00521 TRANSF FDOS DOC - DEST.JOSE LUIS BARRA + + + + +1 +20110630 +-705.00 +30062011 0174628 00000 +0174628 +00521 TRANSF FDOS DOC - DEST.FAGNER H DOS SANTOS LINS + + + + +1 +20110630 +-182.00 +30062011 0229765 00000 +0229765 +00521 TRANSF FDOS DOC - DEST.MARILSA TELES GOMES + + + + +1 +20110630 +-7.80 +30062011 0229765 00000 +0229765 +01963 DOC/TEDINTERNET - DOC INTERNET + + + + +1 +20110630 +-10500.00 +30062011 0001508 00000 +0001508 +00999 CHQ COMPENSADO + + + + +0 +20110701 +182.00 +01072011 0229765 00000 +0229765 +00051 DOC/TED DEVOLV + + + + +0 +20110701 +60021.11 +01072011 7539185 00000 +7539185 +00062 RESG.DE PAPEIS + + + + +1 +20110701 +-6500.00 +01072011 0001512 00000 +0001512 +00002 CHEQUE - ESPECIE + + + + +1 +20110701 +-455.74 +01072011 0000775 00000 +0000775 +00311 PAGTO COBRANCA - LAGOA ENSINO DE IDIOMAS LTDA LUI + + + + +1 +20110701 +-20000.00 +01072011 0291925 00000 +0291925 +00318 TED-T ELET DISP - DEST.ELEVADORES SANBERG LTDA ME + + + + +1 +20110701 +-250.00 +01072011 2832691 00000 +2832691 +00412 TRANSF AUTORIZ - Rodrigo Sousa Xavier + + + + +1 +20110701 +-210.00 +01072011 0248844 00000 +0248844 +00521 TRANSF FDOS DOC - DEST.marcela giovanini g gadelha + + + + +1 +20110701 +-2000.00 +01072011 0459309 00000 +0459309 +00611 TR.AUT.C/C/POUP - Adilson Nascimento dos Santos + + + + +1 +20110701 +-600.00 +01072011 1899884 00000 +1899884 +00611 TR.AUT.C/C/POUP - Zilmar Azevedo dos Santos + + + + +1 +20110701 +-400.00 +01072011 0001518 00000 +0001518 +00783 CH.PAGO OUTR.AG - DIVERSOS RECEBIMENTOS / 3428 + + + + +1 +20110701 +-4833.00 +01072011 0001519 00000 +0001519 +00783 CH.PAGO OUTR.AG - DIVERSOS RECEBIMENTOS / 3176 + + + + +1 +20110701 +-7.80 +01072011 0248844 00000 +0248844 +01963 DOC/TEDINTERNET - DOC INTERNET + + + + +1 +20110701 +-4446.00 +01072011 0001467 00000 +0001467 +00996 CHQ COMPENSADO + + + + +1 +20110701 +-4226.10 +01072011 0001466 00000 +0001466 +00999 CHQ COMPENSADO + + + + +1 +20110701 +-1500.00 +01072011 0001468 00000 +0001468 +00999 CHQ COMPENSADO + + + + +1 +20110701 +-1600.00 +01072011 0001469 00000 +0001469 +00999 CHQ COMPENSADO + + + + +1 +20110701 +-4400.00 +01072011 0001511 00000 +0001511 +00999 CHQ COMPENSADO + + + + +1 +20110701 +-5000.00 +01072011 0001516 00000 +0001516 +00999 CHQ COMPENSADO + + + + +1 +20110701 +-3780.00 +01072011 0001520 00000 +0001520 +00999 CHQ COMPENSADO + + + + +0 +20110704 +210.00 +04072011 0248844 00000 +0248844 +00051 DOC/TED DEVOLV + + + + +1 +20110704 +-5399.34 +04072011 0000771 00000 +0000771 +00311 PAGTO COBRANCA - ALUGUEL VAGA MARINA VEROLME + + + + +1 +20110704 +-222.60 +04072011 0350773 00000 +0350773 +00521 TRANSF FDOS DOC - DEST.MARILSA TELES GOMES + + + + +1 +20110704 +-240.00 +04072011 0375044 00000 +0375044 +00521 TRANSF FDOS DOC - DEST.AMANDA M MAURO + + + + +1 +20110704 +-182.00 +04072011 0381289 00000 +0381289 +00521 TRANSF FDOS DOC - DEST.marilsa teles gomes + + + + +1 +20110704 +-360.00 +04072011 0381735 00000 +0381735 +00521 TRANSF FDOS DOC - DEST.FERNANDA GIOVANINI + + + + +1 +20110704 +-222.60 +04072011 0382274 00000 +0382274 +00521 TRANSF FDOS DOC - DEST.marilsa teles gomes + + + + +1 +20110704 +-250.00 +04072011 1899158 00000 +1899158 +00611 TR.AUT.C/C/POUP - Zilmar Azevedo dos Santos + + + + +1 +20110704 +-6000.00 +04072011 0001513 00000 +0001513 +00999 CHQ COMPENSADO + + + + +1 +20110704 +-1227.04 +04072011 1433341 00000 +1433341 +00932 CONTA TELEFONE - VIVO RJ-01214333416 + + + + + diff --git a/test/ofc_parser.py b/test/ofc_parser.py index c5abc8d..e828a7e 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -14,14 +14,18 @@ no_bankinfo_ofc_path = join(FIXTURES_PATH, 'nobankinfo_and_trnrs.ofc') ofc_with_chknum_path = join(FIXTURES_PATH, 'ofc_with_chknum.ofc') +ofc_with_empty_tag = join(FIXTURES_PATH, 'evil_mailrs.ofx')# its an OFC by inside + def assert_not_raises(function, param, exception): try: function(param) except exception: raise AssertionError, "Exception %s raised" %exception +read_file = lambda f: open(f, 'rU').read() class OFCParserTestCase(unittest.TestCase): + def setUp(self): self.ofc = open(bad_ofc_path, 'r').read() self.parser = OfcParser() @@ -39,6 +43,10 @@ def test_chknum_to_checknum_translation(self): self.assertTrue('CHECKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) self.assertFalse('CHKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) + def test_not_crashes_when_an_OFC_has_empty_tags(self): + ofc = read_file(ofc_with_empty_tag) + assert_not_raises(self.parser.parse, ofc, ParseException) + if __name__ == '__main__': From 26f58876600f0e5eb7b8859aa51b63d91c471819 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 7 Jul 2011 11:26:35 -0300 Subject: [PATCH 18/31] strip_empty_tags is an utility function now. --- lib/ofx/parser.py | 8 ++------ lib/ofxtools/ofc_parser.py | 7 +------ lib/ofxtools/util.py | 8 ++++++++ 3 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 lib/ofxtools/util.py diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index 5a5e63d..9d79c11 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -22,6 +22,7 @@ import sys from pyparsing import alphanums, alphas, CharsNotIn, Dict, Forward, Group, \ Literal, OneOrMore, Optional, SkipTo, White, Word, ZeroOrMore +from ofxtools.util import strip_empty_tags def _ofxStartDebugAction( instring, loc, expr ): sys.stderr.write("Match %s at loc %s (%d,%d)" % @@ -72,18 +73,13 @@ def _tag(self, closed=True): def parse(self, ofx): """Parse a string argument and return a tree structure representing the parsed document.""" - ofx = self.strip_empty_tags(ofx) + ofx = strip_empty_tags(ofx) ofx = self.strip_close_tags(ofx) ofx = self.strip_blank_dtasof(ofx) ofx = self.strip_junk_ascii(ofx) ofx = self.fix_unknown_account_type(ofx) return self.parser.parseString(ofx).asDict() - def strip_empty_tags(self, ofx): - """Strips open/close tags that have no content.""" - strip_search = '<(?P[^>]+)>\s*' - return re.sub(strip_search, '', ofx) - def strip_close_tags(self, ofx): """Strips close tags on non-aggregate nodes. Close tags seem to be valid OFX/1.x, but they screw up our parser definition and are optional. diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 9454927..098dea9 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -54,7 +54,7 @@ def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" ofc = self.remove_inline_closing_tags(ofc) - ofc = self.strip_empty_tags(ofc) + ofc = ofxtools.util.strip_empty_tags(ofc) ofc = self._translate_chknum_to_checknum(ofc) try: return self.parser.parseString(ofc).asDict() @@ -95,11 +95,6 @@ def _translate_chknum_to_checknum(self, ofc): """ return re.sub('CHKNUM', 'CHECKNUM', ofc) - def strip_empty_tags(self, ofc): - """Strips open/close tags that have no content.""" - strip_search = '<(?P[^>]+)>\s*' - return re.sub(strip_search, '', ofc) - def _inject_tags(self, ofc): tags ="\n\n\n0\n0\n0\n\n" if not re.findall(r'\w*\s*', ofc): diff --git a/lib/ofxtools/util.py b/lib/ofxtools/util.py new file mode 100644 index 0000000..3ec94aa --- /dev/null +++ b/lib/ofxtools/util.py @@ -0,0 +1,8 @@ +#coding: utf-8 +import re + +def strip_empty_tags(ofx): + """Strips open/close tags that have no content.""" + strip_search = '<(?P[^>]+)>\s*' + return re.sub(strip_search, '', ofx) + From a36d2ad6c408070affb0915b007c18a7f2833741 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 7 Jul 2011 11:32:01 -0300 Subject: [PATCH 19/31] improving tests readability --- test/fixtures/{evil_mailrs.ofx => empty_tags.ofx} | 0 test/ofc_parser.py | 15 +++++---------- 2 files changed, 5 insertions(+), 10 deletions(-) rename test/fixtures/{evil_mailrs.ofx => empty_tags.ofx} (100%) diff --git a/test/fixtures/evil_mailrs.ofx b/test/fixtures/empty_tags.ofx similarity index 100% rename from test/fixtures/evil_mailrs.ofx rename to test/fixtures/empty_tags.ofx diff --git a/test/ofc_parser.py b/test/ofc_parser.py index e828a7e..70b5bc4 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -10,11 +10,6 @@ import unittest FIXTURES_PATH = join(realpath(dirname(__file__)), 'fixtures') -bad_ofc_path = join(FIXTURES_PATH, 'bad.ofc') -no_bankinfo_ofc_path = join(FIXTURES_PATH, 'nobankinfo_and_trnrs.ofc') -ofc_with_chknum_path = join(FIXTURES_PATH, 'ofc_with_chknum.ofc') - -ofc_with_empty_tag = join(FIXTURES_PATH, 'evil_mailrs.ofx')# its an OFC by inside def assert_not_raises(function, param, exception): try: @@ -22,29 +17,29 @@ def assert_not_raises(function, param, exception): except exception: raise AssertionError, "Exception %s raised" %exception -read_file = lambda f: open(f, 'rU').read() +read_file = lambda f: open(join(FIXTURES_PATH, f), 'rU').read() class OFCParserTestCase(unittest.TestCase): def setUp(self): - self.ofc = open(bad_ofc_path, 'r').read() + self.ofc = read_file('bad.ofc') self.parser = OfcParser() def test_parsing_bad_ofc_should_not_raise_exception(self): assert_not_raises(self.parser.parse, self.ofc, ParseException) def test_parsing_ofc_without_bank_info_not_raise_Exception(self): - self.ofc = open(no_bankinfo_ofc_path, 'r').read() + self.ofc = read_file('nobankinfo_and_trnrs.ofc') assert_not_raises(self.parser.parse, self.ofc, Exception) def test_chknum_to_checknum_translation(self): - self.ofc = open(ofc_with_chknum_path, 'r').read() + self.ofc = read_file('ofc_with_chknum.ofc') #ensure that the CHECKNUM was translated self.assertTrue('CHECKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) self.assertFalse('CHKNUM' in str(self.parser._translate_chknum_to_checknum(self.ofc))) def test_not_crashes_when_an_OFC_has_empty_tags(self): - ofc = read_file(ofc_with_empty_tag) + ofc = read_file('empty_tags.ofx')#it is an ofc by inside assert_not_raises(self.parser.parse, ofc, ParseException) From 321604d4fee3238e7700fae67026e77e5c7a4191 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Thu, 7 Jul 2011 11:49:54 -0300 Subject: [PATCH 20/31] testing for stripping empty tags --- test/test_util.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/test_util.py diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 0000000..737b988 --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,23 @@ +#coding: utf-8 + +import sys +sys.path.insert(0, '../3rdparty') +sys.path.insert(0, '../lib') + +import re +from os.path import join, realpath, dirname +import unittest +from ofxtools import util + + +class StripEmptyTags(unittest.TestCase): + def test_strip_empty_tags(self): + empty_tags_file = open(join(realpath(dirname(__file__)), 'fixtures', 'empty_tags.ofx'), 'rU').read() + empty_tag_pattern = '<(?P[^>]+)>\s*' + + result = util.strip_empty_tags(empty_tags_file) + self.assertFalse(re.match(empty_tag_pattern, result)) + + +if __name__ == '__main__': + unittest.main() From 4d98ae252abe2a2593b600971d5c3c5b1283ab09 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Tue, 9 Aug 2011 12:30:19 -0300 Subject: [PATCH 21/31] fixes problem with max recursion depth limit --- lib/ofxtools/ofc_parser.py | 4 + test/fixtures/recursion_depth_exceeded.ofx | 2020 ++++++++++++++++++++ test/ofc_parser.py | 7 + 3 files changed, 2031 insertions(+) create mode 100644 test/fixtures/recursion_depth_exceeded.ofx diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index 098dea9..ff97c08 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -56,6 +56,10 @@ def parse(self, ofc): ofc = self.remove_inline_closing_tags(ofc) ofc = ofxtools.util.strip_empty_tags(ofc) ofc = self._translate_chknum_to_checknum(ofc) + # if you don't have a good stomach, skip this part + # XXX:needs better solution + import sys + sys.setrecursionlimit(5000) try: return self.parser.parseString(ofc).asDict() except ParseException: diff --git a/test/fixtures/recursion_depth_exceeded.ofx b/test/fixtures/recursion_depth_exceeded.ofx new file mode 100644 index 0000000..eb20518 --- /dev/null +++ b/test/fixtures/recursion_depth_exceeded.ofx @@ -0,0 +1,2020 @@ + + +1566165922 + +20110607 +20110807 +1248.10 + + +1 +20110607 +-80.00 + + +CXE 000034 SAQUE + + + + +1 +20110607 +-14.00 + + +RSHOP-FAR PARK MA-06/06 + + + + +1 +20110607 +-52.80 + + +RSHOP-ITALIA PIZZ-06/06 + + + + +1 +20110607 +-24.50 + + +RSHOP-KILOS -06/06 + + + + +1 +20110608 +-50.00 + + +CXE 003103 SAQUE + + + + +1 +20110608 +-18.15 + + +RSHOP-SUBWAY -07/06 + + + + +1 +20110609 +-65.89 + + +RSHOP-HOOTERS -08/06 + + + + +1 +20110609 +-24.50 + + +RSHOP-KILOS -08/06 + + + + +1 +20110610 +-14.00 + + +RSHOP-FAR PARK MA-09/06 + + + + +1 +20110610 +-65.80 + + +RSHOP-ITALIA PIZZ-09/06 + + + + +1 +20110610 +-24.50 + + +RSHOP-KILOS -09/06 + + + + +1 +20110610 +-8.91 + + +RSHOP-MIKAELA CAF-09/06 + + + + +1 +20110613 +-19.80 + + +RSHOP-ARMAZEM DA -12/06 + + + + +1 +20110613 +-80.00 + + +RSHOP-AUTO POSTO -12/06 + + + + +1 +20110613 +-58.80 + + +RSHOP-CHINA IN BO-11/06 + + + + +1 +20110613 +-6.90 + + +RSHOP-FUNCHAL AUT-10/06 + + + + +1 +20110613 +-24.50 + + +RSHOP-KILOS -10/06 + + + + +1 +20110613 +-8.20 + + +RSHOP-PEDAL PIZZA-12/06 + + + + +1 +20110613 +-18.90 + + +RSHOP-SANTAGOS PI-12/06 + + + + +1 +20110613 +-400.00 + + +DOC INT 526986 + + + + +1 +20110614 +-6.10 + + +RSHOP-CASA DO PAO-13/06 + + + + +1 +20110614 +-14.00 + + +RSHOP-FAR PARK MA-13/06 + + + + +1 +20110614 +-24.50 + + +RSHOP-KILOS -13/06 + + + + +1 +20110614 +-61.04 + + +RSHOP-REQUINTE DE-13/06 + + + + +1 +20110614 +-37.00 + + +RSHOP-TAXI COMUM -13/06 + + + + +1 +20110614 +-116.66 + + +TBI 0393.65025-4 C/C + + + + +1 +20110614 +-371.25 + + +BKI PAG TIT BANCO 237 + + + + +1 +20110614 +-40.00 + + +INT SHOPLINE HOSPEDAGEME + + + + +1 +20110614 +-54.45 + + +SISDEB VALOR ECONOMIC + + + + +1 +20110615 +-24.50 + + +RSHOP-KILOS -14/06 + + + + +1 +20110615 +2476.40 + + +REMUNERACAO/SALARIO + + + + +1 +20110616 +-80.00 + + +CXE 000216 SAQUE + + + + +1 +20110616 +-17.50 + + +RSHOP-CASA DO PAO-15/06 + + + + +1 +20110616 +-24.50 + + +RSHOP-KILOS -15/06 + + + + +1 +20110616 +-7.80 + + +TAR DOC INTERNET + + + + +1 +20110617 +-100.00 + + +CXE 002220 SAQUE + + + + +1 +20110617 +-14.00 + + +RSHOP-FAR PARK MA-16/06 + + + + +1 +20110617 +-53.80 + + +RSHOP-HOOTERS -16/06 + + + + +1 +20110617 +-85.05 + + +RSHOP-HOOTERS -16/06 + + + + +1 +20110617 +-38.36 + + +RSHOP-REQUINTE DE-16/06 + + + + +1 +20110620 +-50.00 + + +CEI 001230 SAQUE 19/06 + + + + +1 +20110620 +-8.20 + + +RSHOP-BAR E RESTA-17/06 + + + + +1 +20110620 +-16.10 + + +RSHOP-CASA DO PAO-19/06 + + + + +1 +20110620 +-19.99 + + +RSHOP-LOJAS AMERI-19/06 + + + + +1 +20110620 +-13.01 + + +RSHOP-MASTER SUPE-19/06 + + + + +1 +20110620 +-174.00 + + +RSHOP-OUTBACK ELD-18/06 + + + + +1 +20110620 +-155.26 + + +RSHOP-OUTBACK HIG-17/06 + + + + +1 +20110620 +-60.69 + + +RSHOP-REQUINTE DE-19/06 + + + + +1 +20110620 +-26.40 + + +RSHOP-STARBUCKS S-18/06 + + + + +1 +20110620 +-45.00 + + +RSHOP-TEATRO FOLH-17/06 + + + + +1 +20110620 +-432.00 + + +RSHOP-TRAXART ELD-18/06 + + + + +1 +20110620 +-33.00 + + +RSHOP-TRIBECA PUB-17/06 + + + + +1 +20110620 +-7.50 + + +RSHOP-UNIBANCO AR-19/06 + + + + +1 +20110620 +-20.00 + + +RSHOP-UNIBANCO AR-19/06 + + + + +1 +20110620 +-10.84 + + +SISDEB SEM PARAR + + + + +1 +20110621 +-80.00 + + +CXE 000109 SAQUE + + + + +1 +20110621 +-14.00 + + +RSHOP-FAR PARK MA-20/06 + + + + +1 +20110621 +-60.80 + + +RSHOP-ITALIA PIZZ-20/06 + + + + +1 +20110621 +-24.50 + + +RSHOP-KILOS -20/06 + + + + +1 +20110622 +-10.50 + + +RSHOP-CASA DO PAO-21/06 + + + + +1 +20110622 +-24.90 + + +RSHOP-ITALIA PIZZ-21/06 + + + + +1 +20110622 +-24.50 + + +RSHOP-KILOS -21/06 + + + + +1 +20110622 +-239.81 + + +CREDICOMP 10/24 + + + + +1 +20110622 +239.81 + + +ESTORNO CREDICOMP + + + + +1 +20110624 +-100.00 + + +CXE 000794 SAQUE + + + + +1 +20110624 +-6.10 + + +RSHOP-CASA DO PAO-22/06 + + + + +1 +20110624 +-14.00 + + +RSHOP-FAR PARK MA-22/06 + + + + +1 +20110624 +-24.50 + + +RSHOP-KILOS -22/06 + + + + +1 +20110624 +-157.51 + + +RSHOP-MASTER SUPE-23/06 + + + + +1 +20110624 +-1356.22 + + +BLOQUEIO JUDICIAL + + + + +1 +20110624 +180.00 + + +CEI 000099 DINHEIRO + + + + +1 +20110624 +20.00 + + +CEI 000100 DINHEIRO + + + + +1 +20110627 +-42.00 + + +RSHOP-CONSULADO D-25/06 + + + + +1 +20110627 +-28.20 + + +RSHOP-DUQUE CENTR-25/06 + + + + +1 +20110627 +-53.70 + + +RSHOP-ITALIA PIZZ-26/06 + + + + +1 +20110627 +-50.00 + + +SAQUE 24H 26660530 + + + + +1 +20110627 +-17.00 + + +INT PRE-PAGO01184112721 + + + + +1 +20110627 +-17.00 + + +INT PRE-PAGO01185048516 + + + + +1 +20110628 +-60.00 + + +CXE 002089 SAQUE + + + + +1 +20110628 +-6.90 + + +RSHOP-FUNCHAL AUT-27/06 + + + + +1 +20110630 +-899.00 + + +TBI 0393.65025-4imps + + + + +1 +20110630 +-2000.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110630 +2433.74 + + +REMUNERACAO/SALARIO + + + + +1 +20110630 +3095.50 + + +REMUNERACAO/SALARIO + + + + +1 +20110701 +-14.00 + + +RSHOP-FAR PARK MA-30/06 + + + + +1 +20110701 +-120.16 + + +TELEFONICA 3257-4602 + + + + +1 +20110701 +0.06 + + +REMUNER BASICA POUP AUT + + + + +1 +20110701 +0.28 + + +JUROS POUP AUT + + + + +1 +20110704 +-60.00 + + +CXE 000372 SAQUE + + + + +1 +20110704 +-80.00 + + +CEI 002238 SAQUE 02/07 + + + + +1 +20110704 +-24.78 + + +RSHOP-DUQUE CENTR-03/07 + + + + +1 +20110704 +-14.00 + + +RSHOP-FAR PARK MA-01/07 + + + + +1 +20110704 +-147.32 + + +RSHOP-LEROY MERLI-02/07 + + + + +1 +20110704 +-94.50 + + +RSHOP-OUTBACK HIG-02/07 + + + + +1 +20110704 +-58.70 + + +RSHOP-PAES E DOCE-03/07 + + + + +1 +20110704 +-20.00 + + +RSHOP-POSTO BOULE-01/07 + + + + +1 +20110704 +-43.10 + + +RSHOP-SUSHIBARA R-03/07 + + + + +1 +20110704 +-1356.22 + + +TRANSF JUDICIAL BCO 001 + + + + +1 +20110704 +1356.22 + + +DESBLOQ TRANSF JUDICIAL + + + + +1 +20110704 +510.00 + + +TBI 0393.65025-4judas + + + + +1 +20110706 +-14.00 + + +RSHOP-FAR PARK MA-05/07 + + + + +1 +20110707 +-11.00 + + +RSHOP-CENTRO AUT -06/07 + + + + +1 +20110707 +-17.89 + + +RSHOP-RODRIGUES &-06/07 + + + + +1 +20110708 +-60.00 + + +CXE 000265 SAQUE + + + + +1 +20110708 +-9.50 + + +RSHOP-CENTRO AUT -07/07 + + + + +1 +20110708 +-14.00 + + +RSHOP-FAR PARK MA-07/07 + + + + +1 +20110708 +-20.00 + + +RSHOP-SKY AUTO PO-07/07 + + + + +1 +20110711 +-60.00 + + +CXE 000232 SAQUE + + + + +1 +20110711 +-144.30 + + +RSHOP-AMERICA PAU-09/07 + + + + +1 +20110711 +-14.00 + + +RSHOP-CASA DO PAO-08/07 + + + + +1 +20110711 +-146.10 + + +RSHOP-DIVINO FOGA-10/07 + + + + +1 +20110711 +-29.00 + + +RSHOP-HAAGEN DAZS-10/07 + + + + +1 +20110711 +-239.92 + + +RSHOP-HERING AV P-09/07 + + + + +1 +20110711 +-75.10 + + +RSHOP-ITALIA PIZZ-09/07 + + + + +1 +20110711 +-130.00 + + +RSHOP-JNE -09/07 + + + + +1 +20110711 +-18.00 + + +RSHOP-JORGE PONTO-09/07 + + + + +1 +20110711 +-38.36 + + +RSHOP-REQUINTE DE-10/07 + + + + +1 +20110711 +-500.00 + + +DOC INT 828625 mamae + + + + +1 +20110711 +-371.24 + + +INT PAG TIT BANCO 237 + + + + +1 +20110712 +-50.00 + + +CXE 000208 SAQUE + + + + +1 +20110712 +-24.50 + + +RSHOP-KILOS -11/07 + + + + +1 +20110712 +-17.00 + + +INT PRE-PAGO01184112721 + + + + +1 +20110712 +-17.00 + + +INT PRE-PAGO01185048516 + + + + +1 +20110712 +-599.80 + + +INT SHOPLINE MOIP.COM + + + + +1 +20110712 +500.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110713 +-24.50 + + +RSHOP-KILOS -12/07 + + + + +1 +20110713 +-7.80 + + +TAR DOC INTERNET + + + + +1 +20110714 +-14.00 + + +RSHOP-FAR PARK MA-13/07 + + + + +1 +20110714 +-17.10 + + +RSHOP-FUNCHAL GRI-13/07 + + + + +1 +20110715 +-24.50 + + +RSHOP-KILOS -14/07 + + + + +1 +20110715 +-66.99 + + +RSHOP-REQUINTE DE-14/07 + + + + +1 +20110715 +-500.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110715 +2476.40 + + +REMUNERACAO/SALARIO + + + + +1 +20110718 +-60.00 + + +CXE 000547 SAQUE + + + + +1 +20110718 +-60.00 + + +CEI 001784 SAQUE 16/07 + + + + +1 +20110718 +-42.80 + + +RSHOP-BURGER KING-16/07 + + + + +1 +20110718 +-13.40 + + +RSHOP-CASA DO PAO-15/07 + + + + +1 +20110718 +-20.00 + + +RSHOP-CENTRO AUT -15/07 + + + + +1 +20110718 +-100.00 + + +RSHOP-CENTRO ODON-17/07 + + + + +1 +20110718 +-84.42 + + +RSHOP-DROGASIL 17-17/07 + + + + +1 +20110718 +-68.00 + + +RSHOP-EMPORIO DAR-17/07 + + + + +1 +20110718 +-14.00 + + +RSHOP-FAR PARK MA-15/07 + + + + +1 +20110718 +-24.50 + + +RSHOP-KILOS -15/07 + + + + +1 +20110718 +-243.58 + + +RSHOP-MASTER SUPE-17/07 + + + + +1 +20110718 +-85.80 + + +RSHOP-PATEO DA LU-17/07 + + + + +1 +20110718 +-319.98 + + +RSHOP-PONTO FRIO -16/07 + + + + +1 +20110718 +-9.50 + + +RSHOP-STAR 008 -16/07 + + + + +1 +20110718 +-200.00 + + +TBI 0393.65025-4 C/C + + + + +1 +20110718 +-400.00 + + +INT DOC 033050210049056 + + + + +1 +20110718 +-26.00 + + +CXE PRE-PAGO01185048516 + + + + +1 +20110718 +-54.45 + + +SISDEB VALOR ECONOMIC + + + + +1 +20110719 +-6.10 + + +RSHOP-CASA DO PAO-18/07 + + + + +1 +20110719 +-24.12 + + +RSHOP-DROGARIA SA-18/07 + + + + +1 +20110719 +-24.50 + + +RSHOP-KILOS -18/07 + + + + +1 +20110720 +-10.40 + + +RSHOP-BAR E RESTA-19/07 + + + + +1 +20110720 +-44.06 + + +RSHOP-FARMALISE -19/07 + + + + +1 +20110720 +-24.50 + + +RSHOP-KILOS -19/07 + + + + +1 +20110720 +-11.90 + + +SISDEB SEM PARAR + + + + +1 +20110720 +-7.80 + + +TAR DOC/TED AGENDADO (I) + + + + +1 +20110721 +-6.10 + + +RSHOP-CASA DO PAO-20/07 + + + + +1 +20110721 +-14.00 + + +RSHOP-FAR PARK MA-20/07 + + + + +1 +20110721 +-24.00 + + +RSHOP-KILOS -20/07 + + + + +1 +20110722 +-20.00 + + +RSHOP-CENTRO AUT -21/07 + + + + +1 +20110722 +-14.00 + + +RSHOP-FAR PARK MA-21/07 + + + + +1 +20110722 +-24.50 + + +RSHOP-KILOS -21/07 + + + + +1 +20110722 +-239.81 + + +CREDICOMP 11/24 + + + + +1 +20110722 +239.81 + + +ESTORNO CREDICOMP + + + + +1 +20110722 +300.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110725 +-50.00 + + +CXE 003202 SAQUE + + + + +1 +20110725 +-6.00 + + +RSHOP-CENTRO AUT -22/07 + + + + +1 +20110725 +-24.50 + + +RSHOP-KILOS -22/07 + + + + +1 +20110725 +-170.00 + + +RSHOP-MASTER SUPE-24/07 + + + + +1 +20110725 +-138.53 + + +SISDEB NET SP + + + + +1 +20110725 +300.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110726 +-100.00 + + +CXE 001206 SAQUE + + + + +1 +20110726 +-19.80 + + +RSHOP-KOPENHAGEN -25/07 + + + + +1 +20110726 +-21.50 + + +RSHOP-MC DONALDS -25/07 + + + + +1 +20110726 +-444.52 + + +INT SHOPLINE VOEAZUL + + + + +1 +20110726 +500.00 + + +TBI 1566.16592-2/500 + + + + +1 +20110727 +-24.50 + + +RSHOP-KILOS -26/07 + + + + +1 +20110728 +-50.00 + + +CXE 000315 SAQUE + + + + +1 +20110728 +-9.90 + + +RSHOP-DUQUE CENTR-27/07 + + + + +1 +20110729 +-833.00 + + +TBI 0393.65025-4imps + + + + +1 +20110729 +-9.99 + + +SISDEB NETMOVIES + + + + +1 +20110729 +2433.57 + + +REMUNERACAO/SALARIO + + + + +1 +20110801 +-300.00 + + +CXE 002568 SAQUE + + + + +1 +20110801 +-70.00 + + +CEI 004432 SAQUE 31/07 + + + + +1 +20110801 +-14.50 + + +RSHOP-DUQUE CENTR-30/07 + + + + +1 +20110801 +-31.10 + + +RSHOP-DUQUE CENTR-30/07 + + + + +1 +20110801 +-6.90 + + +RSHOP-FUNCHAL AUT-29/07 + + + + +1 +20110801 +-77.27 + + +RSHOP-MASTER SUPE-30/07 + + + + +1 +20110801 +-208.39 + + +RSHOP-MASTER SUPE-31/07 + + + + +1 +20110801 +-1648.00 + + +TBI 0393.65025-4 C/C + + + + +1 +20110801 +-48.90 + + +TELEFONICA 3257-4602 + + + + +1 +20110801 +2070.00 + + +TBI 0393.65025-4/500 + + + + +1 +20110801 +0.22 + + +REMUNER BASICA POUP AUT + + + + +1 +20110801 +0.89 + + +JUROS POUP AUT + + + + +1 +20110803 +-89.80 + + +INT SHOPLINE CLICKON + + + + +1 +20110804 +-60.00 + + +CXE 000125 SAQUE + + + + +1 +20110805 +-6.90 + + +RSHOP-FUNCHAL AUT-04/08 + + + + +1 +20110807 +-40.00 + + +CEI 003897 SAQUE 07/08 + + + + + diff --git a/test/ofc_parser.py b/test/ofc_parser.py index 70b5bc4..29811b3 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -42,6 +42,13 @@ def test_not_crashes_when_an_OFC_has_empty_tags(self): ofc = read_file('empty_tags.ofx')#it is an ofc by inside assert_not_raises(self.parser.parse, ofc, ParseException) + def test_not_exceed_max_recursion_limit(self): + """ + For some reason, this file exceeds the normal recursion_limit + i've solved this setting the max recursion depth. + """ + ofc = read_file('recursion_depth_exceeded.ofx') + assert_not_raises(self.parser.parse, ofc, RuntimeError) if __name__ == '__main__': From ffb675178d1f4a0a6126c5de5c78a0c2c94c0ed9 Mon Sep 17 00:00:00 2001 From: rafaeldx7 Date: Fri, 11 Nov 2011 12:30:25 -0200 Subject: [PATCH 22/31] remove blank lines before converting --- fixofx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fixofx.py b/fixofx.py index d5aff30..95203bf 100755 --- a/fixofx.py +++ b/fixofx.py @@ -57,11 +57,13 @@ def fixpath(filename): pass -def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", +def convert(filecontent, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", accttype="UNKNOWN", acctid="UNKNOWN", balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False, debug=False): + text = os.linesep.join(s for s in filecontent.splitlines() if s) + # This finishes a verbosity message started by the caller, where the # caller explains the source command-line option and this explains the # source format. From 4ce1e8dbd94e322cd372a854fb4eaf184fca74bc Mon Sep 17 00:00:00 2001 From: rafaeldx7 Date: Fri, 11 Nov 2011 15:33:56 -0200 Subject: [PATCH 23/31] fix ofc parser. blank tag LEDGR causing error. --- lib/ofxtools/ofc_parser.py | 7 + test/fixtures/invalid_blank_tag_ledger.ofc | 256 +++++++++++++++++++++ test/ofc_parser.py | 4 + 3 files changed, 267 insertions(+) create mode 100644 test/fixtures/invalid_blank_tag_ledger.ofc diff --git a/lib/ofxtools/ofc_parser.py b/lib/ofxtools/ofc_parser.py index ff97c08..8f010cc 100644 --- a/lib/ofxtools/ofc_parser.py +++ b/lib/ofxtools/ofc_parser.py @@ -53,6 +53,7 @@ def _tag(self, closed=True): def parse(self, ofc): """Parse a string argument and return a tree structure representing the parsed document.""" + ofc = self.add_zero_to_empty_ledger_tag(ofc) ofc = self.remove_inline_closing_tags(ofc) ofc = ofxtools.util.strip_empty_tags(ofc) ofc = self._translate_chknum_to_checknum(ofc) @@ -66,6 +67,12 @@ def parse(self, ofc): fixed_ofc = self.fix_ofc(ofc) return self.parser.parseString(fixed_ofc).asDict() + def add_zero_to_empty_ledger_tag(self, ofc): + """ + Fix an OFC, by adding zero to LEDGER blank tag + """ + return re.compile(r'(\D*\n)', re.UNICODE).sub(r'0\1', ofc) + def remove_inline_closing_tags(self, ofc): """ Fix an OFC, by removing inline closing 'tags' diff --git a/test/fixtures/invalid_blank_tag_ledger.ofc b/test/fixtures/invalid_blank_tag_ledger.ofc new file mode 100644 index 0000000..d5406d3 --- /dev/null +++ b/test/fixtures/invalid_blank_tag_ledger.ofc @@ -0,0 +1,256 @@ + +2 +1252 + + +041 +06433500275606 +0 + + +20111101 +20111130 + + +0 +20111101 +800.00 +149109 +149109 +6051 +CR.TRANSFERENCIA +CR.TRANSFERENCIA + + +1 +20111101 +-130.00 +011716 +011716 +6051 +SQ.CASH INT IA +SQ.CASH INT IA + + +1 +20111101 +-847.15 +131523 +131523 +6051 +PG.TITULO +PG.TITULO + + +1 +20111101 +-146.66 +131437 +131437 +6051 +PG. CARTAO CREDI +PG. CARTAO CREDI + + +1 +20111101 +-171.33 +131436 +131436 +6051 +DB.TRANSFERENCIA +DB.TRANSFERENCIA + + +1 +20111101 +-0.60 +000000 +000000 +6051 +IOF +IOF + + +1 +20111101 +-1.16 +000000 +000000 +6051 +IOF ADICIONAL +IOF ADICIONAL + + +5 +20111103 +2115.12 +029355 +029355 +6051 +DEP.CHEQUE - IA +DEP.CHEQUE - IA + + +0 +20111103 +150.00 +149350 +149350 +6051 +CR.TRANSFERENCIA +CR.TRANSFERENCIA + + +1 +20111103 +-10.00 +006478 +006478 +6051 +SAQ.CASH EXTERNO +SAQ.CASH EXTERNO + + +1 +20111103 +-140.00 +008499 +008499 +6051 +SQ.CASH INT IA +SQ.CASH INT IA + + +1 +20111103 +-5.00 +031111 +031111 +6051 +COMPRAS A VISTA +COMPRAS A VISTA + + +1 +20111103 +-33.30 +030911 +030911 +6051 +COMPRAS PRE/PARC +COMPRAS PRE/PARC + + +1 +20111107 +-1675.00 +054520 +054520 +6051 +SAQ. ELETRON-IA +SAQ. ELETRON-IA + + +1 +20111107 +-7.50 +061111 +061111 +6051 +COMPRAS A VISTA +COMPRAS A VISTA + + +1 +20111107 +-5.50 +071111 +071111 +6051 +COMPRAS A VISTA +COMPRAS A VISTA + + +1 +20111108 +-3.50 +081111 +081111 +6051 +COMPRAS A VISTA +COMPRAS A VISTA + + +1 +20111108 +-13.61 +081111 +081111 +6051 +COMPRAS A VISTA +COMPRAS A VISTA + + +1 +20111110 +-58.90 +001827 +001827 +6051 +PG.TELEFONIA/NET +PG.TELEFONIA/NET + + +1 +20111110 +-318.00 +001733 +001733 +6051 +PG.TITULO +PG.TITULO + + +1 +20111111 +-24.90 +121011 +121011 +6051 +COMPRAS PRE/PARC +COMPRAS PRE/PARC + + +1 +20111114 +-30.00 +151011 +151011 +6051 +COMPRAS PRE/PARC +COMPRAS PRE/PARC + + +1 +20111130 +-35.00 +011011 +011011 +6051 +COMPRAS PRE/PARC +COMPRAS PRE/PARC + + +1 +20111202 +-33.30 +030911 +030911 +6051 +COMPRAS PRE/PARC +COMPRAS PRE/PARC + + + + diff --git a/test/ofc_parser.py b/test/ofc_parser.py index 29811b3..912e513 100644 --- a/test/ofc_parser.py +++ b/test/ofc_parser.py @@ -28,6 +28,10 @@ def setUp(self): def test_parsing_bad_ofc_should_not_raise_exception(self): assert_not_raises(self.parser.parse, self.ofc, ParseException) + def test_parsing_ofc_with_blank_ledger_tag_not_raise_Exception(self): + self.ofc = read_file('invalid_blank_tag_ledger.ofc') + assert_not_raises(self.parser.parse, self.ofc, Exception) + def test_parsing_ofc_without_bank_info_not_raise_Exception(self): self.ofc = read_file('nobankinfo_and_trnrs.ofc') assert_not_raises(self.parser.parse, self.ofc, Exception) From 55bfcdba3ebbd16f687b302e832c5260da966a41 Mon Sep 17 00:00:00 2001 From: tinogomes Date: Wed, 22 Jan 2014 15:11:25 -0200 Subject: [PATCH 24/31] fixing broken tests --- test/ofx_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ofx_response.py b/test/ofx_response.py index e2cba10..1ecb36f 100644 --- a/test/ofx_response.py +++ b/test/ofx_response.py @@ -56,7 +56,7 @@ def test_as_xml(self): try: response_elem = ElementTree.fromstring(self.response_text) self.fail("Expected parse exception but did not get one.") - except ExpatError: + except: pass # Then see if we can get a real parse success, with no ExpatError. From 41be9d9fe0472140774ffe80f744fa3cba9de43b Mon Sep 17 00:00:00 2001 From: tinogomes Date: Wed, 22 Jan 2014 15:40:55 -0200 Subject: [PATCH 25/31] importing ofx file with blank memo --- lib/ofx/parser.py | 2 +- test/fixtures/blank_memo.ofx | 123 +++++++++++++++++++++++++++++++++++ test/ofx_parser.py | 16 +++++ test/ofx_test_utils.py | 3 + 4 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/blank_memo.ofx diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index 9d79c11..11dc6b6 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -91,7 +91,7 @@ def strip_close_tags(self, ofx): def strip_blank_dtasof(self, ofx): """Strips empty dtasof tags from wells fargo/wachovia downloads. Again, it would be better to just rewrite the parser, but for now this is a workaround.""" - blank_search = '<(DTASOF|BALAMT|BANKID|CATEGORY|NAME)>[\n\r]+' + blank_search = '<(DTASOF|BALAMT|BANKID|CATEGORY|NAME|MEMO)>[\n\r]+' return re.sub(blank_search, '', ofx) def strip_junk_ascii(self, ofx): diff --git a/test/fixtures/blank_memo.ofx b/test/fixtures/blank_memo.ofx new file mode 100644 index 0000000..bfd5444 --- /dev/null +++ b/test/fixtures/blank_memo.ofx @@ -0,0 +1,123 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + + +0 +INFO + +20140110235959 +POR + + + + +1001 + +0 +INFO + + +BRL + +0341 +5082038208 +CHECKING + + +20140108 +20140110 + +DEBIT +20140108 +-4925.00 +14010801 +14010801 +FIRSTCOM + + +DEBIT +20140108 +-1497.20 +14010802 +14010802 +PERSONNEL SUPPORT + + +DEBIT +20140110 +-12000.00 +14011001 +14011001 +LINCOLN FURLAN ANDO LINCOLN + + +DEBIT +20140110 +-1178.90 +14011002 +14011002 +NUNES E SAWAYA + + +DEBIT +20140110 +-613.95 +14011003 +14011003 +NUNES E SAWAYA + + +DEBIT +20140117 +-722.82 +14011701 +14011701 + + + +DEBIT +20140117 +-3723.90 +14011702 +14011702 +DAVID BRENER + + +DEBIT +20140117 +-464.80 +14011703 +14011703 +REEMBOLSO + + +DEBIT +20140117 +-232.91 +14011704 +14011704 +NUNES E SAWAYA + + +DEBIT +20140110 +-23.26 +14011001 +14011001 +ISS + + + + + + diff --git a/test/ofx_parser.py b/test/ofx_parser.py index b83c58b..d30f74c 100644 --- a/test/ofx_parser.py +++ b/test/ofx_parser.py @@ -27,23 +27,39 @@ def setUp(self): parser = ofx.Parser() checking_stmt = ofx_test_utils.get_checking_stmt() creditcard_stmt = ofx_test_utils.get_creditcard_stmt() + blank_memo_stmt = ofx_test_utils.get_blank_memo_stmt() self.checkparse = parser.parse(checking_stmt) self.creditcardparse = parser.parse(creditcard_stmt) + self.blank_memoparse = parser.parse(blank_memo_stmt) def test_successful_parse(self): """Test parsing a valid OFX document containing a 'success' message.""" self.assertEqual("SUCCESS", self.checkparse["body"]["OFX"]["SIGNONMSGSRSV1"]["SONRS"]["STATUS"]["MESSAGE"]) + def test_successfull_parse_for_blank_memo(self): + """Test parsing a valid OFX document with blank memo containing a 'success' message.""" + self.assertEqual("INFO", + self.blank_memoparse["body"]["OFX"]["SIGNONMSGSRSV1"]["SONRS"]["STATUS"]["SEVERITY"]) + def test_body_read(self): """Test reading a value from deep in the body of the OFX document.""" self.assertEqual("-5128.16", self.creditcardparse["body"]["OFX"]["CREDITCARDMSGSRSV1"]["CCSTMTTRNRS"]["CCSTMTRS"]["LEDGERBAL"]["BALAMT"]) + def test_body_read_for_blank_memo(self): + """Test reading a value from deep in the body of the OFX document.""" + self.assertEqual("-23.26", + self.blank_memoparse["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"]["BANKTRANLIST"]["STMTTRN"]["TRNAMT"]) + def test_header_read(self): """Test reading a header from the OFX document.""" self.assertEqual("100", self.checkparse["header"]["OFXHEADER"]) + def test_header_read_for_blank_memo(self): + """Test reading a header from the OFX document.""" + self.assertEqual("100", self.blank_memoparse["header"]["OFXHEADER"]) + if __name__ == '__main__': unittest.main() diff --git a/test/ofx_test_utils.py b/test/ofx_test_utils.py index 0101774..77802b7 100644 --- a/test/ofx_test_utils.py +++ b/test/ofx_test_utils.py @@ -25,6 +25,9 @@ def get_savings_stmt(): def get_creditcard_stmt(): return _read_file("creditcard.ofx") +def get_blank_memo_stmt(): + return _read_file("blank_memo.ofx") + def _read_file(filename): return open(os.path.join(fixtures, filename), 'rU').read() From 82e6f54647b3a84b00f0dc9b45f885a1efe8f714 Mon Sep 17 00:00:00 2001 From: tinogomes Date: Wed, 22 Jan 2014 20:35:43 -0200 Subject: [PATCH 26/31] import category from qif files --- lib/ofx/builder.py | 3 ++- lib/ofxtools/qif_converter.py | 32 ++++++++++++++++++++------------ lib/ofxtools/qif_parser.py | 1 + test/ofxtools_qif_converter.py | 4 ---- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/ofx/builder.py b/lib/ofx/builder.py index b7f160b..809bdaa 100644 --- a/lib/ofx/builder.py +++ b/lib/ofx/builder.py @@ -122,7 +122,7 @@ def __call__(self, *values, **params): # also be added here, unfortunately. __all__ = ['ACCTID', 'ACCTINFORQ', 'ACCTINFOTRNRQ', 'ACCTTYPE', 'APPID', 'APPVER', 'AVAILBAL', 'BALAMT', 'BANKACCTFROM', 'BANKID', 'BANKMSGSRQV1', -'BANKMSGSRSV1', 'BANKTRANLIST', 'BROKERID', 'CCACCTFROM', 'CCSTMTENDRQ', +'BANKMSGSRSV1', 'BANKTRANLIST', 'BROKERID', 'CATEGORY', 'CCACCTFROM', 'CCSTMTENDRQ', 'CCSTMTENDTRNRQ', 'CCSTMTRQ', 'CCSTMTRS', 'CCSTMTTRNRQ', 'CCSTMTTRNRS', 'CHARSET', 'CHECKNUM', 'CLIENTROUTING', 'CLTCOOKIE', 'CODE', 'COMPRESSION', 'CREDITCARDMSGSRQV1', 'CREDITCARDMSGSRSV1', 'CURDEF', 'DATA', 'DOCUMENT', @@ -153,6 +153,7 @@ def __call__(self, *values, **params): BANKMSGSRSV1 = Tag("BANKMSGSRSV1", aggregate=True) BANKTRANLIST = Tag("BANKTRANLIST", aggregate=True) BROKERID = Tag("BROKERID") +CATEGORY = Tag("CATEGORY") CCACCTFROM = Tag("CCACCTFROM", aggregate=True) CCSTMTENDRQ = Tag("CCSTMTENDRQ", aggregate=True) CCSTMTENDTRNRQ = Tag("CCSTMTENDTRNRQ", aggregate=True) diff --git a/lib/ofxtools/qif_converter.py b/lib/ofxtools/qif_converter.py index c9f2d25..7b019a1 100644 --- a/lib/ofxtools/qif_converter.py +++ b/lib/ofxtools/qif_converter.py @@ -370,12 +370,13 @@ def _clean_txn_number(self, txn): txn["Type"] = "CHECK" def _clean_txn_type(self, txn): - txn_type = "UNKNOWN" - txn_amount = txn.get("Amount", "UNKNOWN") - txn_payee = txn.get("Payee", "UNKNOWN") - txn_memo = txn.get("Memo", "UNKNOWN") - txn_number = txn.get("Number", "UNKNOWN") - txn_sign = self._txn_sign(txn_amount) + txn_type = "UNKNOWN" + txn_amount = txn.get("Amount", "UNKNOWN") + txn_payee = txn.get("Payee", "UNKNOWN") + txn_memo = txn.get("Memo", "UNKNOWN") + txn_category = txn.get("Category", "UNKNOWN") + txn_number = txn.get("Number", "UNKNOWN") + txn_sign = self._txn_sign(txn_amount) # Try to figure out the transaction type from the Payee or # Memo field. @@ -401,12 +402,13 @@ def _clean_txn_type(self, txn): break def _clean_txn_payee(self, txn): - txn_payee = txn.get("Payee", "UNKNOWN") - txn_memo = txn.get("Memo", "UNKNOWN") - txn_number = txn.get("Number", "UNKNOWN") - txn_type = txn.get("Type", "UNKNOWN") - txn_amount = txn.get("Amount", "UNKNOWN") - txn_sign = self._txn_sign(txn_amount) + txn_payee = txn.get("Payee", "UNKNOWN") + txn_memo = txn.get("Memo", "UNKNOWN") + txn_category = txn.get("Category", "UNKNOWN") + txn_number = txn.get("Number", "UNKNOWN") + txn_type = txn.get("Type", "UNKNOWN") + txn_amount = txn.get("Amount", "UNKNOWN") + txn_sign = self._txn_sign(txn_amount) # Try to fill in the payee field with some meaningful value. if txn_payee == "UNKNOWN": @@ -436,6 +438,9 @@ def _clean_txn_payee(self, txn): elif txn_memo != "UNKNOWN": txn["Payee"] = txn_memo + elif txn_category != "UNKNOWN": + txn["Payee"] = txn_category + # Down here, we have no payee, no memo, no check number, # and no type. Who knows what this stuff is. elif txn_type == "UNKNOWN" and txn_sign == "debit": @@ -618,6 +623,9 @@ def _ofx_txn(self, txn): if self._check_field("Memo", txn): fields.append(MEMO(sax.escape(sax.unescape(txn["Memo"].strip())))) + if self._check_field("Category", txn): + fields.append(CATEGORY(sax.escape(sax.unescape(txn["Category"].strip())))) + return STMTTRN(*fields) def _check_field(self, key, txn): diff --git a/lib/ofxtools/qif_parser.py b/lib/ofxtools/qif_parser.py index 62027e6..fcdbfa3 100644 --- a/lib/ofxtools/qif_parser.py +++ b/lib/ofxtools/qif_parser.py @@ -113,6 +113,7 @@ def __init__(self, debug=False): self.parser = Group(ZeroOrMore(White()).suppress() + ZeroOrMore(acctlist).suppress() + OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns | invsttxns) + + ZeroOrMore(category | classlist).suppress() + ZeroOrMore(White()).suppress() ).setResultsName("QifStatement") diff --git a/test/ofxtools_qif_converter.py b/test/ofxtools_qif_converter.py index 1c512d7..5d4645a 100644 --- a/test/ofxtools_qif_converter.py +++ b/test/ofxtools_qif_converter.py @@ -23,9 +23,6 @@ from time import localtime, strftime class QifConverterTests(unittest.TestCase): - def setUp(self): - pass - def test_bank_stmttype(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -324,6 +321,5 @@ def test_check_stmt_number(self): txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn.get("Type"), "CHECK") - if __name__ == '__main__': unittest.main() From fa66467af104d3c9f7dc96d63e2426e111ee0507 Mon Sep 17 00:00:00 2001 From: leandrost Date: Thu, 17 Sep 2015 18:48:48 -0300 Subject: [PATCH 27/31] Fix parser error when OFX contains a self closed tag --- lib/ofxtools/util.py | 2 +- .../savings_with_self_closed_empty_tag.ofx | 42 +++++++++++++++++++ test/ofx_parser.py | 30 +++++++++---- test/ofx_test_utils.py | 11 +++-- 4 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 test/fixtures/savings_with_self_closed_empty_tag.ofx diff --git a/lib/ofxtools/util.py b/lib/ofxtools/util.py index 3ec94aa..b5bd103 100644 --- a/lib/ofxtools/util.py +++ b/lib/ofxtools/util.py @@ -3,6 +3,6 @@ def strip_empty_tags(ofx): """Strips open/close tags that have no content.""" - strip_search = '<(?P[^>]+)>\s*' + strip_search = '(<(?P[^>]+)>\s*|<(?P[^>]+)/>)' return re.sub(strip_search, '', ofx) diff --git a/test/fixtures/savings_with_self_closed_empty_tag.ofx b/test/fixtures/savings_with_self_closed_empty_tag.ofx new file mode 100644 index 0000000..37175bb --- /dev/null +++ b/test/fixtures/savings_with_self_closed_empty_tag.ofx @@ -0,0 +1,42 @@ +OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:NONE ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE + + + + 0 + INFO + + 20150908120000[-3:BRT] + POR + + Fake Bank + 1 + + + + + + 1 + + 0 + INFO + + + BRL + + 1 + + / + SAVINGS + + + 20150902120000[-3:BRT] + 20150902120000[-3:BRT] + + + 640,70 + 20150908120000[-3:BRT] + + + + + diff --git a/test/ofx_parser.py b/test/ofx_parser.py index d30f74c..9c85662 100644 --- a/test/ofx_parser.py +++ b/test/ofx_parser.py @@ -1,11 +1,11 @@ # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,12 +31,12 @@ def setUp(self): self.checkparse = parser.parse(checking_stmt) self.creditcardparse = parser.parse(creditcard_stmt) self.blank_memoparse = parser.parse(blank_memo_stmt) - + def test_successful_parse(self): """Test parsing a valid OFX document containing a 'success' message.""" self.assertEqual("SUCCESS", self.checkparse["body"]["OFX"]["SIGNONMSGSRSV1"]["SONRS"]["STATUS"]["MESSAGE"]) - + def test_successfull_parse_for_blank_memo(self): """Test parsing a valid OFX document with blank memo containing a 'success' message.""" self.assertEqual("INFO", @@ -46,20 +46,32 @@ def test_body_read(self): """Test reading a value from deep in the body of the OFX document.""" self.assertEqual("-5128.16", self.creditcardparse["body"]["OFX"]["CREDITCARDMSGSRSV1"]["CCSTMTTRNRS"]["CCSTMTRS"]["LEDGERBAL"]["BALAMT"]) - + def test_body_read_for_blank_memo(self): """Test reading a value from deep in the body of the OFX document.""" self.assertEqual("-23.26", self.blank_memoparse["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"]["BANKTRANLIST"]["STMTTRN"]["TRNAMT"]) - + def test_header_read(self): """Test reading a header from the OFX document.""" self.assertEqual("100", self.checkparse["header"]["OFXHEADER"]) - + def test_header_read_for_blank_memo(self): """Test reading a header from the OFX document.""" self.assertEqual("100", self.blank_memoparse["header"]["OFXHEADER"]) - + + def test_parse_with_empty_tag(self): + """Test reading a header from the OFX document.""" + parser = ofx.Parser() + empty_tag_stmt = \ + ofx_test_utils.get_savings_with_self_closed_empty_tag_stmt() + self.empty_tag = parser.parse(empty_tag_stmt) + bank_acc_from = self.empty_tag["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"]["BANKACCTFROM"] + self.assertEqual(bank_acc_from['BANKID'], '1') + self.assertEqual(bank_acc_from['ACCTID'], '/') + self.assertEqual(bank_acc_from['ACCTTYPE'], 'SAVINGS') + self.assertEqual(['ACCTID', 'ACCTTYPE', 'BANKID'], bank_acc_from.keys()) + if __name__ == '__main__': unittest.main() diff --git a/test/ofx_test_utils.py b/test/ofx_test_utils.py index 77802b7..8711b35 100644 --- a/test/ofx_test_utils.py +++ b/test/ofx_test_utils.py @@ -1,11 +1,11 @@ # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,10 +18,13 @@ def get_checking_stmt(): return _read_file("checking.ofx") - + def get_savings_stmt(): return _read_file("savings.ofx") +def get_savings_with_self_closed_empty_tag_stmt(): + return _read_file("savings_with_self_closed_empty_tag.ofx") + def get_creditcard_stmt(): return _read_file("creditcard.ofx") From 05f3b6ee46a2d949307322854650780e69926d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Thim=C3=B3teo?= Date: Tue, 25 Sep 2018 12:18:33 -0300 Subject: [PATCH 28/31] Add test to tag with line break --- lib/ofx/parser.py | 5 +++ test/fixtures/tag_with_line_break.ofx | 62 +++++++++++++++++++++++++++ test/ofx_parser.py | 12 ++++++ test/ofx_test_utils.py | 3 ++ 4 files changed, 82 insertions(+) create mode 100644 test/fixtures/tag_with_line_break.ofx diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index 11dc6b6..14d26f8 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -74,10 +74,15 @@ def parse(self, ofx): """Parse a string argument and return a tree structure representing the parsed document.""" ofx = strip_empty_tags(ofx) + print 0 ofx = self.strip_close_tags(ofx) + print 1 ofx = self.strip_blank_dtasof(ofx) + print 2 ofx = self.strip_junk_ascii(ofx) + print 3 ofx = self.fix_unknown_account_type(ofx) + print 4 return self.parser.parseString(ofx).asDict() def strip_close_tags(self, ofx): diff --git a/test/fixtures/tag_with_line_break.ofx b/test/fixtures/tag_with_line_break.ofx new file mode 100644 index 0000000..0ae5a23 --- /dev/null +++ b/test/fixtures/tag_with_line_break.ofx @@ -0,0 +1,62 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + + 0 + INFO + + 20180924120000[-3:BRT] + POR + + Banco Cooperativo do Brasil + 756 + + + + + + 1 + + 0 + INFO + + + BRL + + 756 + 1711-3 + 99000-5 + CHECKING + + + 20180901120000[-3:BRT] + 20180924120000[-3:BRT] + + DEBIT + 20180903120000[-3:BRT] + -150.00 + 20180903150001 + 2582963 + 2582963 + DÉB.TRANSF.CONTAS DIF.TITULARIDADE + crazy string + the same in the toddynho + + + + 38633.35 + 20180924120000[-3:BRT] + + + + + diff --git a/test/ofx_parser.py b/test/ofx_parser.py index 9c85662..f6820a1 100644 --- a/test/ofx_parser.py +++ b/test/ofx_parser.py @@ -28,6 +28,8 @@ def setUp(self): checking_stmt = ofx_test_utils.get_checking_stmt() creditcard_stmt = ofx_test_utils.get_creditcard_stmt() blank_memo_stmt = ofx_test_utils.get_blank_memo_stmt() + tag_with_line_break_stmt = ofx_test_utils.get_tag_with_line_break_stmt() + self.checkparse = parser.parse(checking_stmt) self.creditcardparse = parser.parse(creditcard_stmt) self.blank_memoparse = parser.parse(blank_memo_stmt) @@ -72,6 +74,16 @@ def test_parse_with_empty_tag(self): self.assertEqual(bank_acc_from['ACCTTYPE'], 'SAVINGS') self.assertEqual(['ACCTID', 'ACCTTYPE', 'BANKID'], bank_acc_from.keys()) + def test_parse_tag_with_line_break(self): + """Test reading a header from the OFX document.""" + parser = ofx.Parser() + stmt = ofx_test_utils.get_tag_with_line_break_stmt() + result = parser.parse(stmt) + subject = result["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"] + ["STMTRS"]["BANKTRANLIST"]["STMTTRN"]["OBSV"] + + self.assertEqual(subject, "crazy string with \n the same in the toddynho") + if __name__ == '__main__': unittest.main() diff --git a/test/ofx_test_utils.py b/test/ofx_test_utils.py index 8711b35..610c849 100644 --- a/test/ofx_test_utils.py +++ b/test/ofx_test_utils.py @@ -31,6 +31,9 @@ def get_creditcard_stmt(): def get_blank_memo_stmt(): return _read_file("blank_memo.ofx") +def get_tag_with_line_break_stmt(): + return _read_file("tag_with_line_break.ofx") + def _read_file(filename): return open(os.path.join(fixtures, filename), 'rU').read() From 9345c3b3836c2a33bcfe1db34b3f8b29de33f839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Thim=C3=B3teo?= Date: Tue, 25 Sep 2018 17:51:56 -0300 Subject: [PATCH 29/31] Strips line break from OBSV tags --- lib/ofx/parser.py | 11 ++++++----- test/fixtures/tag_with_line_break.ofx | 15 ++++++++++++++- test/ofx_parser.py | 13 ++++++------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index 14d26f8..a46c9ec 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -74,15 +74,11 @@ def parse(self, ofx): """Parse a string argument and return a tree structure representing the parsed document.""" ofx = strip_empty_tags(ofx) - print 0 ofx = self.strip_close_tags(ofx) - print 1 ofx = self.strip_blank_dtasof(ofx) - print 2 ofx = self.strip_junk_ascii(ofx) - print 3 ofx = self.fix_unknown_account_type(ofx) - print 4 + ofx = self.strip_obsv_tag(ofx) return self.parser.parseString(ofx).asDict() def strip_close_tags(self, ofx): @@ -110,3 +106,8 @@ def fix_unknown_account_type(self, ofx): parser is able to parse it. This isn't really the best solution, but it's a decent workaround.""" return re.sub('(?P[<\n\r])', 'UNKNOWN\g', ofx) + def strip_obsv_tag(self, ofx): + """Strips OBSV tags. This is a workaround.""" + strip_search = '.*?' + return re.sub(strip_search, '', ofx, flags=re.DOTALL) + diff --git a/test/fixtures/tag_with_line_break.ofx b/test/fixtures/tag_with_line_break.ofx index 0ae5a23..9bfe7bd 100644 --- a/test/fixtures/tag_with_line_break.ofx +++ b/test/fixtures/tag_with_line_break.ofx @@ -48,7 +48,20 @@ NEWFILEUID:NONE 2582963 2582963 DÉB.TRANSF.CONTAS DIF.TITULARIDADE - crazy string + crazy string 1 + the same in the toddynho + ever + + + + DEBIT + 20180903120000[-3:BRT] + -150.00 + 20180903150001 + 2582963 + 2582963 + DÉB.TRANSF.CONTAS DIF.TITULARIDADE + crazy string 2 the same in the toddynho diff --git a/test/ofx_parser.py b/test/ofx_parser.py index f6820a1..8641ebd 100644 --- a/test/ofx_parser.py +++ b/test/ofx_parser.py @@ -26,12 +26,12 @@ class ParserTests(unittest.TestCase): def setUp(self): parser = ofx.Parser() checking_stmt = ofx_test_utils.get_checking_stmt() - creditcard_stmt = ofx_test_utils.get_creditcard_stmt() - blank_memo_stmt = ofx_test_utils.get_blank_memo_stmt() - tag_with_line_break_stmt = ofx_test_utils.get_tag_with_line_break_stmt() - self.checkparse = parser.parse(checking_stmt) + + creditcard_stmt = ofx_test_utils.get_creditcard_stmt() self.creditcardparse = parser.parse(creditcard_stmt) + + blank_memo_stmt = ofx_test_utils.get_blank_memo_stmt() self.blank_memoparse = parser.parse(blank_memo_stmt) def test_successful_parse(self): @@ -79,10 +79,9 @@ def test_parse_tag_with_line_break(self): parser = ofx.Parser() stmt = ofx_test_utils.get_tag_with_line_break_stmt() result = parser.parse(stmt) - subject = result["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"] - ["STMTRS"]["BANKTRANLIST"]["STMTTRN"]["OBSV"] + subject = result["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"]["BANKTRANLIST"]["STMTTRN"] - self.assertEqual(subject, "crazy string with \n the same in the toddynho") + self.assertEqual(False, "OBSV" in subject) if __name__ == '__main__': From a0920ce6038d1576e9672b071d5569956ba0f30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Thim=C3=B3teo?= Date: Thu, 27 Sep 2018 10:33:31 -0300 Subject: [PATCH 30/31] Fix multiline tags --- lib/ofx/parser.py | 25 +- test/fixtures/tag_with_line_break.ofx | 1195 +++++++++++++++++++++++-- test/ofx_parser.py | 6 +- 3 files changed, 1156 insertions(+), 70 deletions(-) diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index a46c9ec..d588c4a 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -20,6 +20,7 @@ import re import sys +from StringIO import StringIO from pyparsing import alphanums, alphas, CharsNotIn, Dict, Forward, Group, \ Literal, OneOrMore, Optional, SkipTo, White, Word, ZeroOrMore from ofxtools.util import strip_empty_tags @@ -74,11 +75,11 @@ def parse(self, ofx): """Parse a string argument and return a tree structure representing the parsed document.""" ofx = strip_empty_tags(ofx) + ofx = self.fix_multiline_tags(ofx) ofx = self.strip_close_tags(ofx) ofx = self.strip_blank_dtasof(ofx) ofx = self.strip_junk_ascii(ofx) ofx = self.fix_unknown_account_type(ofx) - ofx = self.strip_obsv_tag(ofx) return self.parser.parseString(ofx).asDict() def strip_close_tags(self, ofx): @@ -106,8 +107,24 @@ def fix_unknown_account_type(self, ofx): parser is able to parse it. This isn't really the best solution, but it's a decent workaround.""" return re.sub('(?P[<\n\r])', 'UNKNOWN\g', ofx) - def strip_obsv_tag(self, ofx): + def fix_multiline_tags(self, ofx): """Strips OBSV tags. This is a workaround.""" - strip_search = '.*?' - return re.sub(strip_search, '', ofx, flags=re.DOTALL) + buf = StringIO(ofx) + out = StringIO() + tag = '' + + for line in buf: + if re.match('^\s*<[^/]', line): + tagClose = line.index('>') + 1 + tag = line[:tagClose] + else: + if tag: + strip_search = '^\s*(?!\s*<)' + if line.strip() != '' and re.match(strip_search, line): + line = tag + line + + out.write(line) + + return out.getvalue() + diff --git a/test/fixtures/tag_with_line_break.ofx b/test/fixtures/tag_with_line_break.ofx index 9bfe7bd..99dd1d8 100644 --- a/test/fixtures/tag_with_line_break.ofx +++ b/test/fixtures/tag_with_line_break.ofx @@ -8,68 +8,1135 @@ COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE - - - - 0 - INFO - - 20180924120000[-3:BRT] - POR - - Banco Cooperativo do Brasil - 756 - - - - - - 1 - - 0 - INFO - - - BRL - - 756 - 1711-3 - 99000-5 - CHECKING - - - 20180901120000[-3:BRT] - 20180924120000[-3:BRT] - - DEBIT - 20180903120000[-3:BRT] - -150.00 - 20180903150001 - 2582963 - 2582963 - DÉB.TRANSF.CONTAS DIF.TITULARIDADE - crazy string 1 - the same in the toddynho - ever - - - - DEBIT - 20180903120000[-3:BRT] - -150.00 - 20180903150001 - 2582963 - 2582963 - DÉB.TRANSF.CONTAS DIF.TITULARIDADE - crazy string 2 - the same in the toddynho - - - - 38633.35 - 20180924120000[-3:BRT] - - - - + + + +0 +INFO + +20180928120000[-3:BRT] +POR + +xxxxxxxx +171 + + + + + +1 + +0 +INFO + + +BRL + +171 +2456-3 +12455-5 +CHECKING + + +20180901120000[-3:BRT] +20180924120000[-3:BRT] + +DEBIT +20180903120000[-3:BRT] +-1006.00 +201809031006001 +2582961 +2582961 +DÉB.TIT.COMPE EFETIVADO +copos personalizados + + +DEBIT +20180903120000[-3:BRT] +-150.00 +20180903150001 +2582963 +2582963 +DÉB.TRANSF.CONTAS DIF.TITULARIDADE +FAV.: JL NICOLAO - ME + frete cohouse para scheneider + + +DEBIT +20180903120000[-3:BRT] +-11.55 +2018090311551 +0000016859 +0000016859 +TARIFA COBRANÇA + + +CREDIT +20180903120000[-3:BRT] +4875.00 +201809034875001 +0000016859 +0000016859 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +CREDIT +20180904120000[-3:BRT] +530.00 +20180904530001 +94458854 +94458854 +CRÉD.TED-STR +ADRIANA SORAIA SCHUCH + 077.381.409-41 + CODIGO TED: T310464635 + 00000000000 + + +DEBIT +20180904120000[-3:BRT] +-1300.00 +201809041300001 +2586000 +2586000 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Roberta Maria de oliveira + limpeza evento hoje 2018 + + +DEBIT +20180904120000[-3:BRT] +-679.00 +20180904679001 +2586001 +2586001 +DÉB.TIT.COMPE EFETIVADO +funil pro + + +DEBIT +20180904120000[-3:BRT] +-4399.49 +201809044399491 +2586002 +2586002 +DEBITO EMISSÃO TED DIF.TITULARIDADE +NC comunicacoes + pagamento ric e atlantica + + +DEBIT +20180904120000[-3:BRT] +-6.60 +201809046601 +0000006115 +0000006115 +TARIFA COBRANÇA + + +CREDIT +20180904120000[-3:BRT] +556.67 +20180904556671 +0000006115 +0000006115 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180904120000[-3:BRT] +-8.00 +201809048001 +188 +188 +TED INTERNET + + +DEBIT +20180905120000[-3:BRT] +-3500.00 +201809053500001 +2590261 +2590261 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Gefferson vivan + salario gefferson + + +DEBIT +20180905120000[-3:BRT] +-2222.01 +201809052222011 +2590262 +2590262 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Hiago soares + salario hiago curso e decimo + + +DEBIT +20180905120000[-3:BRT] +-1250.00 +201809051250001 +2590263 +2590263 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Lucas dutra + salario lucas + + +CREDIT +20180905120000[-3:BRT] +2926.48 +201809052926481 +0000004129 +0000004129 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180905120000[-3:BRT] +-18.15 +2018090518151 +0000004129 +0000004129 +TARIFA COBRANÇA + + +DEBIT +20180905120000[-3:BRT] +-12.00 +2018090512001 +188 +188 +TED INTERNET + + +CREDIT +20180906120000[-3:BRT] +1000.00 +201809061000001 +174 +174 +DEP.DINHEIRO +ENVELOPE: 0403335136 + + +CREDIT +20180906120000[-3:BRT] +1500.00 +201809061500001 +176 +176 +DEP.DINHEIRO +ENVELOPE: 0193567088 + + +CREDIT +20180906120000[-3:BRT] +1500.00 +201809061500002 +177 +177 +DEP.DINHEIRO +ENVELOPE: 0193567096 + + +CREDIT +20180906120000[-3:BRT] +1000.00 +201809061000002 +185 +185 +DEP.DINHEIRO +ENVELOPE: 0193567070 + + +CREDIT +20180906120000[-3:BRT] +1033.66 +201809061033661 +0000010643 +0000010643 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180906120000[-3:BRT] +-16.50 +2018090616501 +0000010643 +0000010643 +TARIFA COBRANÇA + + +CREDIT +20180910120000[-3:BRT] +3507.02 +201809103507021 +94829931 +94829931 +CRÉD.TED-STR +@WIRECARD BRASIL S.A + 08.718.431 0001-08 + CODIGO TED: T312756039 + + +CREDIT +20180910120000[-3:BRT] +9996.50 +201809109996501 +94830008 +94830008 +CRÉD.TED-STR +@WIRECARD BRASIL S.A + 08.718.431 0001-08 + CODIGO TED: T312756452 + + +CREDIT +20180910120000[-3:BRT] +1111.50 +201809101111501 +3067 +3067 +CRED.TRANSF.CONTAS INTERCREDIS +REM.: ACESSOLINE TELECOMUNICAÇ + + +CREDIT +20180910120000[-3:BRT] +6128.64 +201809106128641 +0000004365 +0000004365 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180910120000[-3:BRT] +-47.85 +2018091047851 +0000004365 +0000004365 +TARIFA COBRANÇA + + +CREDIT +20180911120000[-3:BRT] +8530.18 +201809118530181 +94978118 +94978118 +CRÉD.TED-STR +@WIRECARD BRASIL S.A + 08.718.431 0001-08 + CODIGO TED: T313691748 + + +DEBIT +20180911120000[-3:BRT] +-93.59 +2018091193591 +2610528 +2610528 +DÉB.TIT.COMPE EFETIVADO +PASSARELA CENTER + + +DEBIT +20180911120000[-3:BRT] +-83.49 +2018091183491 +2610529 +2610529 +DÉB.CONV.ORGÃOS GOV. +SEGURANÇA PUBLICA + + +DEBIT +20180911120000[-3:BRT] +-75.00 +2018091175001 +2610530 +2610530 +DÉB.CONV.ORGÃOS GOV. +SEGURANÇA PUBLICA + + +DEBIT +20180911120000[-3:BRT] +-2800.00 +201809112800001 +2610531 +2610531 +DÉB.TRANSF.CONTAS DIF.TIT. INTERCREDIS +FAV.: BTM COMÉRCIO DE BRINDES LTDA + btm brindes + + +DEBIT +20180911120000[-3:BRT] +-857.49 +20180911857491 +2610532 +2610532 +DÉB.TIT.COMPE EFETIVADO +nova turismo + + +DEBIT +20180911120000[-3:BRT] +-18.15 +2018091118151 +0000019395 +0000019395 +TARIFA COBRANÇA + + +CREDIT +20180911120000[-3:BRT] +1303.13 +201809111303131 +0000019395 +0000019395 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +CREDIT +20180912120000[-3:BRT] +85000.00 +2018091285000001 +513258 +513258 +CRÉD.EMPRÉSTIMO + + +DEBIT +20180912120000[-3:BRT] +-848.00 +20180912848001 +2614196 +2614196 +DÉB.TIT.COMPE EFETIVADO +PULSEIRAS + + +DEBIT +20180912120000[-3:BRT] +-16.50 +2018091216501 +0000011108 +0000011108 +TARIFA COBRANÇA + + +CREDIT +20180912120000[-3:BRT] +2717.16 +201809122717161 +0000011108 +0000011108 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180913120000[-3:BRT] +-7908.67 +201809137908671 +00449811 +00449811 +DÉB.EMPRÉSTIMO + + +DEBIT +20180913120000[-3:BRT] +-19600.00 +2018091319600001 +2616614 +2616614 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Audace solucoes + palestrante clovis filho + + +DEBIT +20180913120000[-3:BRT] +-7200.00 +201809137200001 +2616615 +2616615 +DEBITO EMISSÃO TED DIF.TITULARIDADE +The first consultoria + palestra diogenes + + +DEBIT +20180913120000[-3:BRT] +-2953.33 +201809132953331 +2616616 +2616616 +DÉB.TÍTULO COBRANÇA +camistas almofadas + + +DEBIT +20180913120000[-3:BRT] +-315.00 +20180913315001 +2616617 +2616617 +DÉB.TRANSF.CONTAS DIF.TITULARIDADE +FAV.: VALDINEI CLIMACO VARELA EIRELI + valdir entrega panfletos + + +DEBIT +20180913120000[-3:BRT] +-16.50 +2018091316501 +0000005106 +0000005106 +TARIFA COBRANÇA + + +CREDIT +20180913120000[-3:BRT] +2468.97 +201809132468971 +0000005106 +0000005106 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180913120000[-3:BRT] +-8.00 +201809138001 +188 +188 +TED INTERNET + + +DEBIT +20180914120000[-3:BRT] +-8000.00 +201809148000001 +2618577 +2618577 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Ciro botini + Ciro botini + + +CREDIT +20180914120000[-3:BRT] +68.00 +2018091468001 +3067 +3067 +DEP.DINHEIRO - INTERCREDIS +NOME: JAQUELINE KLEIN + + +DEBIT +20180914120000[-3:BRT] +-20000.00 +2018091420000001 +2619537 +2619537 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Rodrigo mega + Mega producoes + + +DEBIT +20180914120000[-3:BRT] +-418.00 +20180914418001 +2619653 +2619653 +DEBITO EMISSÃO TED DIF.TITULARIDADE +tailana fotografa + pagamento 1 parcela fotografa + + +CREDIT +20180914120000[-3:BRT] +418.00 +20180914418002 +0 +DEVOLTED +CRÉDITO-DEVOLUÇÃO TED DIF.TITULARIDADE +CODIGO TED: 314627012 + NOME: tailana fotografa + CPF: 05660257950 + MOTIVO: AG CONTA INVÁLIDA + + +DEBIT +20180914120000[-3:BRT] +-5200.00 +201809145200001 +2619889 +2619889 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Pontual piso + piso evento hoje + + +CREDIT +20180914120000[-3:BRT] +1969.33 +201809141969331 +0000016200 +0000016200 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180914120000[-3:BRT] +-23.10 +2018091423101 +0000016200 +0000016200 +TARIFA COBRANÇA + + +DEBIT +20180914120000[-3:BRT] +-16.00 +2018091416001 +188 +188 +TED INTERNET + + +CREDIT +20180917120000[-3:BRT] +404.00 +20180917404001 +149 +149 +DEP.DINHEIRO +ENVELOPE: 0403119811 + + +CREDIT +20180917120000[-3:BRT] +1330.00 +201809171330001 +95350950 +95350950 +CRÉD.TED-STR +COOPERATIVA DE PRODUCAO E CONSUMO CONCOR + 83.573.212 0001-95 + CODIGO TED: T314889928 + + +CREDIT +20180917120000[-3:BRT] +142.00 +20180917142001 +2623031 +2623031 +CRÉD.TRANSF.CONTAS +REM.: HELLEN SARILA SOUZA RIBEIRO + + +DEBIT +20180917120000[-3:BRT] +-756.00 +20180917756001 +2624932 +2624932 +DÉB.TÍTULO COBRANÇA +nobre + + +DEBIT +20180917120000[-3:BRT] +-2500.00 +201809172500001 +2624933 +2624933 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Aline jones + palestra aline jones + + +DEBIT +20180917120000[-3:BRT] +-5000.00 +201809175000001 +2624934 +2624934 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Juliana munaro + palestra juliana munaro + + +DEBIT +20180917120000[-3:BRT] +-5868.90 +201809175868901 +2624935 +2624935 +DÉB.TIT.COMPE EFETIVADO +aluguel do parque + + +DEBIT +20180917120000[-3:BRT] +-14000.00 +2018091714000001 +2624936 +2624936 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Banda principal hoje18 + banda surpresa + + +DEBIT +20180917120000[-3:BRT] +-418.00 +20180917418001 +2624937 +2624937 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Leonardo Mantoani + adiantamento fotografa tailana + + +CREDIT +20180917120000[-3:BRT] +600.00 +20180917600001 +3067 +3067 +DEP.DINHEIRO - INTERCREDIS +NOME: ATM GRAZIELA + ENVELOPE: 0408046951 + + +CREDIT +20180917120000[-3:BRT] +600.00 +20180917600002 +3067 +3067 +DEP.DINHEIRO - INTERCREDIS +NOME: ATM GRAZIELA + ENVELOPE: 0408046969 + + +CREDIT +20180917120000[-3:BRT] +418.00 +20180917418002 +0 +DEVOLTED +CRÉDITO-DEVOLUÇÃO TED DIF.TITULARIDADE +CODIGO TED: 315080100 + NOME: Leonardo Mantoani + CPF: 05660257950 + MOTIVO: AG CONTA INVÁLIDA + + +CREDIT +20180917120000[-3:BRT] +10242.16 +2018091710242161 +0000004627 +0000004627 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180917120000[-3:BRT] +-37.95 +2018091737951 +0000004627 +0000004627 +TARIFA COBRANÇA + + +DEBIT +20180917120000[-3:BRT] +-16.00 +2018091716001 +188 +188 +TED INTERNET + + +CREDIT +20180918120000[-3:BRT] +2030.51 +201809182030511 +95465752 +95465752 +CRÉD.TED-STR +@WIRECARD BRASIL S.A + 08.718.431 0001-08 + CODIGO TED: T315307885 + + +DEBIT +20180918120000[-3:BRT] +-1930.00 +201809181930001 +2628467 +2628467 +DÉB.TRANSF.CONTAS DIF.TIT. INTERCREDIS +FAV.: INOVE FABRICAÇÃO DE MOVEIS SOB MED + inove decoração + + +CREDIT +20180918120000[-3:BRT] +5920.77 +201809185920771 +0000008126 +0000008126 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180918120000[-3:BRT] +-19.80 +2018091819801 +0000008126 +0000008126 +TARIFA COBRANÇA + + +DEBIT +20180919120000[-3:BRT] +-2531.78 +201809192531781 +0 +MASTERCARD +DÉB.CONV.DEMAIS EMPRESAS + + +DEBIT +20180919120000[-3:BRT] +-500.00 +20180919500001 +4 +4 +SAQUE NA AGENCIA + + +DEBIT +20180919120000[-3:BRT] +-5500.00 +201809195500001 +2631417 +2631417 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Sandro roncado + tentente cascavel + + +DEBIT +20180919120000[-3:BRT] +-13.20 +2018091913201 +0000019238 +0000019238 +TARIFA COBRANÇA + + +CREDIT +20180919120000[-3:BRT] +3752.63 +201809193752631 +0000019238 +0000019238 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180919120000[-3:BRT] +-4.00 +201809194001 +188 +188 +TED INTERNET + + +DEBIT +20180920120000[-3:BRT] +-2350.00 +201809202350001 +2631418 +2631418 +DÉB.TIT.COMPE EFETIVADO +locação de transporte + + +DEBIT +20180920120000[-3:BRT] +-750.00 +20180920750001 +2631419 +2631419 +DÉB.TIT.COMPE EFETIVADO +moblee + + +DEBIT +20180920120000[-3:BRT] +-5476.29 +201809205476291 +2631420 +2631420 +DÉB.TIT.COMPE EFETIVADO +ecad + + +DEBIT +20180920120000[-3:BRT] +-2482.74 +201809202482741 +2631421 +2631421 +DÉB.CONV.TRIBUTOS FEDERAIS - RFB +simples + + +DEBIT +20180920120000[-3:BRT] +-104.94 +20180920104941 +2631422 +2631422 +DÉBITO PAGAMENTO AGENDADO GPS + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080061 +2631660 +2631660 +DÉB.TIT.COMPE EFETIVADO +taxa fiscalização + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080062 +2631661 +2631661 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080063 +2631662 +2631662 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia sanitaria + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080064 +2631663 +2631663 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia sanitaria + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080065 +2631664 +2631664 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia sanitaria + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080066 +2631665 +2631665 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia sanitaria + + +DEBIT +20180920120000[-3:BRT] +-80.06 +2018092080067 +2631696 +2631696 +DÉB.TIT.COMPE EFETIVADO +taxa vigilancia sanitaria + + +DEBIT +20180920120000[-3:BRT] +-25.00 +2018092025001 +2632826 +2632826 +DÉB.CONV.ORGÃOS GOV. + + +DEBIT +20180920120000[-3:BRT] +-29.49 +2018092029491 +2632832 +2632832 +DÉB.CONV.ORGÃOS GOV. + + +DEBIT +20180920120000[-3:BRT] +-7500.00 +201809207500001 +2633205 +2633205 +DEBITO EMISSÃO TED DIF.TITULARIDADE +Classe a transporte + Transporte banda + + +DEBIT +20180920120000[-3:BRT] +-91.50 +2018092091501 +2635045 +2635045 +DÉB.TIT.COMPE EFETIVADO + + +CREDIT +20180920120000[-3:BRT] +7306.59 +201809207306591 +0000018211 +0000018211 +CRÉD.LIQUIDAÇÃO COBRANÇA + + +DEBIT +20180920120000[-3:BRT] +-21.45 +2018092021451 +0000018211 +0000018211 +TARIFA COBRANÇA + + +DEBIT +20180920120000[-3:BRT] +-4.00 +201809204001 +188 +188 +TED INTERNET + + +DEBIT +20180921120000[-3:BRT] +-91.50 +2018092191501 +2635537 +2635537 +DÉB.TIT.COMPE EFETIVADO + + +DEBIT +20180921120000[-3:BRT] +-91.50 +2018092191502 +2635538 +2635538 +DÉB.TIT.COMPE EFETIVADO + + +CREDIT +20180921120000[-3:BRT] +948.85 +20180921948851 +76423199 +76423199 +CR COMPRAS MAESTRO +SIPAG_Deb._Maestro + + +DEBIT +20180921120000[-3:BRT] +-80.06 +2018092180061 +2635776 +2635776 +DÉB.TIT.COMPE EFETIVADO + + +CREDIT +20180921120000[-3:BRT] +440.28 +20180921440281 +95715372 +95715372 +CRÉD.TED-STR +SERVICO NACIONAL DE APRENDIZAGEM INDUSTR + 03.774.688 0001-55 + CODIGO TED: T316146171 + PAGAMENTOS DIVERSOS + + +CREDIT +20180924120000[-3:BRT] +4789.26 +201809244789261 +77042189 +77042189 +CR COMPRAS MAESTRO +SIPAG_Deb._Maestro + + +CREDIT +20180924120000[-3:BRT] +451.41 +20180924451411 +77042190 +77042190 +CR COMPRAS VISA ELECTRON +SIPAG_Deb._Visa Electron + + +CREDIT +20180924120000[-3:BRT] +366.51 +20180924366511 +77042191 +77042191 +CR COMPRAS DEB OUTRAS BANDEIRAS +SIPAG_Deb._Elo Débito + + + +38633.35 +20180924120000[-3:BRT] + + + + diff --git a/test/ofx_parser.py b/test/ofx_parser.py index 8641ebd..2953f57 100644 --- a/test/ofx_parser.py +++ b/test/ofx_parser.py @@ -79,9 +79,11 @@ def test_parse_tag_with_line_break(self): parser = ofx.Parser() stmt = ofx_test_utils.get_tag_with_line_break_stmt() result = parser.parse(stmt) - subject = result["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"]["BANKTRANLIST"]["STMTTRN"] + subject = result["body"]["OFX"]["BANKMSGSRSV1"]["STMTTRNRS"]["STMTRS"] \ + ["BANKTRANLIST"] - self.assertEqual(False, "OBSV" in subject) + transactions_count = len([e[0] for e in subject if e[0] == 'STMTTRN']) + self.assertEqual(109, transactions_count) if __name__ == '__main__': From 03631944c3e2f7827939ed9cd4683c7de8186775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Thim=C3=B3teo?= Date: Fri, 28 Sep 2018 11:47:26 -0300 Subject: [PATCH 31/31] Refactoring regexp strings --- lib/ofx/parser.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/ofx/parser.py b/lib/ofx/parser.py index d588c4a..ef12261 100644 --- a/lib/ofx/parser.py +++ b/lib/ofx/parser.py @@ -87,25 +87,25 @@ def strip_close_tags(self, ofx): valid OFX/1.x, but they screw up our parser definition and are optional. This allows me to keep using the same parser without having to re-write it from scratch just yet.""" - strip_search = '<(?P[^>]+)>\s*(?P[^<\n\r]+)(?:\s*)?(?P[\n\r]*)' + strip_search = r'<(?P[^>]+)>\s*(?P[^<\n\r]+)(?:\s*)?(?P[\n\r]*)' return re.sub(strip_search, '<\g>\g\g', ofx) def strip_blank_dtasof(self, ofx): """Strips empty dtasof tags from wells fargo/wachovia downloads. Again, it would be better to just rewrite the parser, but for now this is a workaround.""" - blank_search = '<(DTASOF|BALAMT|BANKID|CATEGORY|NAME|MEMO)>[\n\r]+' + blank_search = r'<(DTASOF|BALAMT|BANKID|CATEGORY|NAME|MEMO)>[\n\r]+' return re.sub(blank_search, '', ofx) def strip_junk_ascii(self, ofx): """Strips high ascii gibberish characters from Schwab statements. They seem to contains strings of EF BF BD EF BF BD 0A 08 EF BF BD 64 EF BF BD in the field, and the newline is screwing up the parser.""" - return re.sub('[\xBD-\xFF\x64\x0A\x08]{4,}', '', ofx) + return re.sub(r'[\xBD-\xFF\x64\x0A\x08]{4,}', '', ofx) def fix_unknown_account_type(self, ofx): """Sets the content of nodes without content to be UNKNOWN so that the parser is able to parse it. This isn't really the best solution, but it's a decent workaround.""" - return re.sub('(?P[<\n\r])', 'UNKNOWN\g', ofx) + return re.sub(r'(?P[<\n\r])', 'UNKNOWN\g', ofx) def fix_multiline_tags(self, ofx): """Strips OBSV tags. This is a workaround.""" @@ -114,13 +114,13 @@ def fix_multiline_tags(self, ofx): tag = '' for line in buf: - if re.match('^\s*<[^/]', line): + if re.match(r'^\s*<[^/]', line): tagClose = line.index('>') + 1 tag = line[:tagClose] else: if tag: - strip_search = '^\s*(?!\s*<)' - if line.strip() != '' and re.match(strip_search, line): + strip_search = r'^\s*(?!\s*<)' + if line.strip() and re.match(strip_search, line): line = tag + line out.write(line)