diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index e8def1ee..9eedbdf9 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -67,7 +67,7 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/README.md b/README.md index 307037b5..e71b7ed4 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ #### Run Server + ```bash ./scripts/up.sh ``` diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 88fe3908..fbdea0ac 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -17,9 +17,9 @@ services: - POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password - POSTGRES_EXTRA_OPTS=-Z9 --schema=public --blobs - SCHEDULE=@daily - - BACKUP_KEEP_DAYS=30 - - BACKUP_KEEP_WEEKS=4 - - BACKUP_KEEP_MONTHS=6 + - BACKUP_KEEP_DAYS=10 + - BACKUP_KEEP_WEEKS=2 + - BACKUP_KEEP_MONTHS=3 - HEALTHCHECK_PORT=80 deploy: replicas: 1 diff --git a/requirements.txt b/requirements.txt index a4e8fe74..66cd418a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,4 +38,6 @@ SQLAlchemy-Utils~=0.38.2 ujson~=5.2.0 web3~=5.13.1 Werkzeug~=1.0.1 -markupsafe==2.0.1 \ No newline at end of file +markupsafe==2.0.1 +urllib3 + diff --git a/say/api/child_api.py b/say/api/child_api.py index b1f5a23f..f41a6d48 100644 --- a/say/api/child_api.py +++ b/say/api/child_api.py @@ -928,7 +928,7 @@ def patch(self, child_id): if family: family.isDeleted = True - child.social_worker.currentChildCount -= 1 + child.social_worker.current_child_count -= 1 child.ngo.currentChildrenCount -= 1 safe_commit(session) diff --git a/say/api/need_api.py b/say/api/need_api.py index 0c9ffeb4..6d79626d 100644 --- a/say/api/need_api.py +++ b/say/api/need_api.py @@ -296,8 +296,8 @@ def patch(self, need_id): return need_dict if 'cost' in request.form.keys(): + print("updating cost...") new_cost = int(request.form['cost'].replace(',', '')) - if ( ( sw_role in [SOCIAL_WORKER, COORDINATOR, NGO_SUPERVISOR] @@ -329,12 +329,12 @@ def patch(self, need_id): if 'link' in request.form.keys(): new_link = request.form['link'] - if new_link != need.link: - from say.tasks import update_need - - need.link = new_link - session.flush() - update_need.delay(need.id, force=True) + from say.tasks import update_need + print("updating link...") + need.link = new_link + session.flush() + update_need.delay(need.id) + if 'affiliateLinkUrl' in request.form.keys(): need.affiliateLinkUrl = request.form['affiliateLinkUrl'] @@ -691,9 +691,9 @@ def post(self): if new_need.link: from say.tasks import update_need - update_need.delay(new_need.id) + return new_need diff --git a/say/api/preneed_api.py b/say/api/preneed_api.py index 3457b3c3..2a9608c8 100644 --- a/say/api/preneed_api.py +++ b/say/api/preneed_api.py @@ -30,6 +30,7 @@ def get(self): ).filter( Need.child_id == DEFAULT_CHILD_ID, Need.isDeleted.is_(False), + ~Need.imageUrl.ilike('%wrong path%') # Case-insensitive ) return PreneedSummarySchema.from_query_list(preneeds) diff --git a/say/celery.py b/say/celery.py index 20e5330e..2b083b72 100644 --- a/say/celery.py +++ b/say/celery.py @@ -23,7 +23,7 @@ }, 'update-needs': { 'task': 'say.tasks.update_needs.update_needs', - 'schedule': crontab(minute=30, hour='0,4,8,12,16,20'), + 'schedule': crontab(minute=30, hour='6'), }, 'report_to_family': { 'task': 'say.tasks.report_to_family.report_to_families', @@ -37,10 +37,10 @@ 'task': 'say.tasks.delivere_to_child.delivere_to_child', 'schedule': crontab(minute='10,40'), }, - 'update_nakama_txs': { - 'task': 'say.tasks.nakama.update_nakama_txs', - 'schedule': 10 * 60, - }, + # 'update_nakama_txs': { + # 'task': 'say.tasks.nakama.update_nakama_txs', + # 'schedule': 10 * 60, + # }, 'check_unverified_payments': { 'task': 'say.tasks.check_unverified_payments.check_unverified_payments', 'schedule': crontab(minute=59), diff --git a/say/config.py b/say/config.py index 42d1d5bd..b4ca8327 100644 --- a/say/config.py +++ b/say/config.py @@ -73,7 +73,7 @@ class Config(object): # Celery BROKER = "redis" - task_soft_time_limit = 60 + task_soft_time_limit = 120 task_acks_late = True worker_prefetch_multiplier = 1 diff --git a/say/crawler/__init__.py b/say/crawler/__init__.py index 0840c386..2fc16915 100644 --- a/say/crawler/__init__.py +++ b/say/crawler/__init__.py @@ -2,10 +2,12 @@ import html import re from typing import NamedTuple - +import urllib.request +import json import requests from cachetools import TTLCache from cachetools import cached +from urllib.parse import urljoin from say.config import configs from say.crawler.patterns import get_patterns @@ -105,6 +107,7 @@ def get_data(self, force=False): class DigikalaCrawler: + PROXY = "https://proxy.saydao.org" API_URL_NOT_FRESH = 'https://api.digikala.com/v2/product/%s/' API_URL_FRESH = 'https://api-fresh.digikala.com/v1/product/%s/' DKP_PATTERN = re.compile(r'.*/dkp-(\d+).*') @@ -115,37 +118,76 @@ def __init__(self, url): except IndexError: self.dkp = None - def get_data(self, force=False): + def call_api(self, url): + try: + print("updating via server...") + with urllib.request.urlopen(url) as response: + status_code = response.getcode() + content = response.read().decode('utf-8') + print("updated via server...") + return status_code, content + except urllib.error.URLError as e: + # If there's an error, use proxy + try: + print("updating via proxy...") + proxy_url = urljoin(self.PROXY, "proxy?url=%s/") % url + with urllib.request.urlopen(proxy_url) as proxy_response: + proxy_status_code = proxy_response.getcode() + proxy_content = proxy_response.read().decode('utf-8') + print("updated via proxy...") + return proxy_status_code, proxy_content + except urllib.error.URLError as proxy_error: + return None, f"An error occurred with both APIs: {e} and {proxy_error}" + + def parse_result(self, api_response): + try: + # Parse the JSON response + json_response = json.loads(api_response[1]) + return json_response + except json.JSONDecodeError as e: + return f"An error occurred while parsing JSON: {e}" + + def get_data(self, force): + result = None + parsed_result = None + if self.dkp is None: return - - url = self.API_URL_NOT_FRESH % self.dkp - if force: - r = requests.get(url) - else: - r = request_with_cache(url) - if r.status_code != 200: - # fresh products have different api + url = self.API_URL_NOT_FRESH % self.dkp + api_response = self.call_api(url) + parsed_result = self.parse_result(api_response) + if int(parsed_result["status"]) == 200: + parsed_result = self.parse_result(api_response) + elif parsed_result["status"] == 302 and "fresh" in parsed_result["redirect_url"]["uri"]: url = self.API_URL_FRESH % self.dkp - if force: - r = requests.get(url) + api_response = self.call_api(url) + parsed_result = self.parse_result(api_response) + if parsed_result["status"] != 200: + print("Could not update!") + return else: - r = request_with_cache(url) + parsed_result = self.parse_result(api_response) - if r.status_code != 200: - return + else: + print("Could not update!") + print(url) + return - data = r.json()['data'] + result = parsed_result["data"] - if data['product'].get('is_inactive'): + + if result['product'].get('is_inactive'): return dict(cost='unavailable', img=None, title=None) - title = data['product']['title_fa'] - if data['product']['status'] == 'marketable': - cost = int(data['product']['default_variant']['price']['rrp_price']) // 10 + title = result['product']['title_fa'] + if result['product']['status'] == 'marketable': + cost = int(result['product']['default_variant']['price']['rrp_price']) // 10 else: cost = 'unavailable' - img = data['product']['images']['main']['url'][0] + img = result['product']['images']['main']['url'][0] return dict(cost=cost, img=img, title=title) + + + diff --git a/say/models/need_model.py b/say/models/need_model.py index 88b6a04b..1209c105 100644 --- a/say/models/need_model.py +++ b/say/models/need_model.py @@ -447,17 +447,22 @@ def update(self, force=False): from say.crawler import DigikalaCrawler if 'digikala' in self.link: + print("updating via crawler.....") data = DigikalaCrawler(self.link).get_data(force=force) + print("Done updating.") else: data = Crawler(self.link).get_data(force=force) if data is None: + print(f"Could not get data for: {self.id}") return img = data['img'] title = data['title'] cost = data['cost'] - + print(f"Id: {self.id}") + print(f"fetched title: {title}") + print(f"fetched cost: {cost}") if img: self.img = img diff --git a/say/models/social_worker_model.py b/say/models/social_worker_model.py index 81fed409..1ccd28fb 100644 --- a/say/models/social_worker_model.py +++ b/say/models/social_worker_model.py @@ -203,18 +203,20 @@ def validate_password(self, password): def send_password(self, password): from say.authorization import create_sw_access_token from say.tasks import send_embeded_subject_email - - send_embeded_subject_email.delay( - to=self.email, - html=render_template_i18n( - 'social_worker_password.html', - social_worker=self, - surname=surname(self.gender), - password=password, - token=create_sw_access_token(self), - locale=self.locale, - ), - ) + try: + send_embeded_subject_email.delay( + to=self.email, + html=render_template_i18n( + 'social_worker_password.html', + social_worker=self, + surname=surname(self.gender), + password=password, + token=create_sw_access_token(self), + locale=self.locale, + ), + ) + except Exception as e: + self.retry(exc=e, countdown=60) @staticmethod def generate_password(): diff --git a/say/static/images/github.png b/say/static/images/github.png new file mode 100644 index 00000000..402191e0 Binary files /dev/null and b/say/static/images/github.png differ diff --git a/say/tasks/update_needs.py b/say/tasks/update_needs.py index 03eb8ea8..b3487889 100644 --- a/say/tasks/update_needs.py +++ b/say/tasks/update_needs.py @@ -1,6 +1,6 @@ from time import sleep -from sqlalchemy import or_ +from sqlalchemy import or_, and_ from say.celery import celery from say.orm import safe_commit @@ -12,17 +12,19 @@ def update_needs(self): needs = self.session.query(Need).filter( Need.type == 1, - or_( - Need.status < 4, - Need.title.is_(None), - ), + Need.status < 3, Need.isDeleted.is_(False), Need.link.isnot(None), ) t = [] + counter = 0 + print(f"Total needs to be updated: {needs.count()}") for need in needs: + counter+=1 t.append(need.id) + print(f"{counter}/{needs.count()}-> updating need: {need.id}") + sleep(10) update_need.delay(need.id) return t @@ -38,9 +40,7 @@ def update_needs(self): ) def update_need(self, need_id, force=False): from say.models.need_model import Need - - sleep(5) - need = self.session.query(Need).get(need_id) + need = self.session.query(Need).get(need_id) data = need.update(force=force) safe_commit(self.session) diff --git a/say/templates/en/base.html b/say/templates/en/base.html index 0b765ffe..8b29c0c6 100644 --- a/say/templates/en/base.html +++ b/say/templates/en/base.html @@ -140,7 +140,7 @@ >
- +

What you hear is real

Building a platform to share wealth and love, literally

diff --git a/say/templates/en/successful_payment.html b/say/templates/en/successful_payment.html index 2ab86ad9..eb0b4626 100644 --- a/say/templates/en/successful_payment.html +++ b/say/templates/en/successful_payment.html @@ -106,7 +106,7 @@