From a8e89c91ffe9a0fb95f0be5e52858ee08dbccd41 Mon Sep 17 00:00:00 2001 From: Alexander Zubkov Date: Sun, 18 Dec 2022 14:28:00 +0300 Subject: [PATCH 1/2] Use index and increase array limit Works much faster and fixes unknown error when database becomes large. --- postfixparser/webui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postfixparser/webui.py b/postfixparser/webui.py index 04b551b..2725a1e 100644 --- a/postfixparser/webui.py +++ b/postfixparser/webui.py @@ -157,7 +157,7 @@ async def api_emails(): _sm = await _process_filters(query=_sm, frm=frm) _sm, res = await _paginate_query(_sm, frm, rt_conn=conn, rt_query=r_q, order_by=order_by, order_dir=order_dir) - _sm = await _sm.run(conn) + _sm = await _sm.run(conn, array_limit=10000000) sm = [] if type(_sm) is list: @@ -197,7 +197,7 @@ async def _paginate_query(query: QueryOrTable, frm: Mapping, rt_conn: DefaultCon # if order_by in dict(settings.rethink_tables)['sent_mail']: # _sm = _sm.order_by(index=r_order) # else: - query = query.order_by(r_order).skip(offset).limit(limit) + query = query.order_by(index=r_order).skip(offset).limit(limit) return query, res From d772983f352abdea54b678da2fd5d78dfc5efd8f Mon Sep 17 00:00:00 2001 From: Alexander Zubkov Date: Sun, 18 Dec 2022 17:04:13 +0300 Subject: [PATCH 2/2] Fix searching --- postfixparser/webui.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/postfixparser/webui.py b/postfixparser/webui.py index 2725a1e..9755d0b 100644 --- a/postfixparser/webui.py +++ b/postfixparser/webui.py @@ -39,6 +39,8 @@ RqlQuery = rethinkdb.query.ast.RqlQuery QueryOrTable = Union[Table, RqlQuery] +skip_keys = ['limit', 'offset', 'page'] + @app.route('/', methods=['GET']) async def index(): @@ -197,16 +199,18 @@ async def _paginate_query(query: QueryOrTable, frm: Mapping, rt_conn: DefaultCon # if order_by in dict(settings.rethink_tables)['sent_mail']: # _sm = _sm.order_by(index=r_order) # else: - query = query.order_by(index=r_order).skip(offset).limit(limit) + if empty(frm.keys() - skip_keys, itr=True): + query = query.order_by(index=r_order) + else: + query = query.order_by(r_order) + query = query.skip(offset).limit(limit) return query, res -async def _process_filters(query: QueryOrTable, frm: Mapping, skip_keys: List[str] = None) -> QueryOrTable: +async def _process_filters(query: QueryOrTable, frm: Mapping) -> QueryOrTable: if empty(frm, itr=True): return query - skip_keys = ['limit', 'offset', 'page'] if not skip_keys else skip_keys - for fkey, fval in frm.items(): if fkey in skip_keys: continue