Release

.github/workflows/pipeline.yml

@@ -67,7 +67,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

README.md

@@ -12,6 +12,7 @@
 
 #### Run Server
 
+
 ```bash
 ./scripts/up.sh
 ```

docker-compose-prod.yml

@@ -17,9 +17,9 @@ services:
       - POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password
       - POSTGRES_EXTRA_OPTS=-Z9 --schema=public --blobs
       - SCHEDULE=@daily
-      - BACKUP_KEEP_DAYS=30
-      - BACKUP_KEEP_WEEKS=4
-      - BACKUP_KEEP_MONTHS=6
+      - BACKUP_KEEP_DAYS=10
+      - BACKUP_KEEP_WEEKS=2
+      - BACKUP_KEEP_MONTHS=3
       - HEALTHCHECK_PORT=80
     deploy:
       replicas: 1

requirements.txt

@@ -38,4 +38,6 @@ SQLAlchemy-Utils~=0.38.2
 ujson~=5.2.0
 web3~=5.13.1
 Werkzeug~=1.0.1
-markupsafe==2.0.1
+markupsafe==2.0.1
+urllib3
+

say/api/child_api.py

@@ -928,7 +928,7 @@ def patch(self, child_id):
         if family:
             family.isDeleted = True
 
-        child.social_worker.currentChildCount -= 1
+        child.social_worker.current_child_count -= 1
         child.ngo.currentChildrenCount -= 1
 
         safe_commit(session)

say/api/need_api.py

@@ -296,8 +296,8 @@ def patch(self, need_id):
             return need_dict
 
         if 'cost' in request.form.keys():
+            print("updating cost...")
             new_cost = int(request.form['cost'].replace(',', ''))
-
             if (
                 (
                     sw_role in [SOCIAL_WORKER, COORDINATOR, NGO_SUPERVISOR]
@@ -329,12 +329,12 @@ def patch(self, need_id):
 
         if 'link' in request.form.keys():
             new_link = request.form['link']
-            if new_link != need.link:
-                from say.tasks import update_need
-
-                need.link = new_link
-                session.flush()
-                update_need.delay(need.id, force=True)
+            from say.tasks import update_need
+            print("updating link...")
+            need.link = new_link
+            session.flush()
+            update_need.delay(need.id)
+
 
         if 'affiliateLinkUrl' in request.form.keys():
             need.affiliateLinkUrl = request.form['affiliateLinkUrl']
@@ -691,9 +691,9 @@ def post(self):
 
         if new_need.link:
             from say.tasks import update_need
-
             update_need.delay(new_need.id)
 
+
         return new_need
 
 

say/api/preneed_api.py

@@ -30,6 +30,7 @@ def get(self):
         ).filter(
             Need.child_id == DEFAULT_CHILD_ID,
             Need.isDeleted.is_(False),
+            ~Need.imageUrl.ilike('%wrong path%')  # Case-insensitive
         )
 
         return PreneedSummarySchema.from_query_list(preneeds)

say/celery.py

@@ -23,7 +23,7 @@
     },
     'update-needs': {
         'task': 'say.tasks.update_needs.update_needs',
-        'schedule': crontab(minute=30, hour='0,4,8,12,16,20'),
+        'schedule': crontab(minute=30, hour='6'),
     },
     'report_to_family': {
         'task': 'say.tasks.report_to_family.report_to_families',
@@ -37,10 +37,10 @@
         'task': 'say.tasks.delivere_to_child.delivere_to_child',
         'schedule': crontab(minute='10,40'),
     },
-    'update_nakama_txs': {
-        'task': 'say.tasks.nakama.update_nakama_txs',
-        'schedule': 10 * 60,
-    },
+    # 'update_nakama_txs': {
+    #     'task': 'say.tasks.nakama.update_nakama_txs',
+    #     'schedule': 10 * 60,
+    # },
     'check_unverified_payments': {
         'task': 'say.tasks.check_unverified_payments.check_unverified_payments',
         'schedule': crontab(minute=59),

say/config.py

@@ -73,7 +73,7 @@ class Config(object):
 
     # Celery
     BROKER = "redis"
-    task_soft_time_limit = 60
+    task_soft_time_limit = 120
     task_acks_late = True
     worker_prefetch_multiplier = 1
 

say/crawler/__init__.py

@@ -2,10 +2,12 @@
 import html
 import re
 from typing import NamedTuple
-
+import urllib.request
+import json
 import requests
 from cachetools import TTLCache
 from cachetools import cached
+from urllib.parse import urljoin
 
 from say.config import configs
 from say.crawler.patterns import get_patterns
@@ -105,6 +107,7 @@ def get_data(self, force=False):
 
 
 class DigikalaCrawler:
+    PROXY = "https://proxy.saydao.org"  
     API_URL_NOT_FRESH = 'https://api.digikala.com/v2/product/%s/'
     API_URL_FRESH = 'https://api-fresh.digikala.com/v1/product/%s/'
     DKP_PATTERN = re.compile(r'.*/dkp-(\d+).*')
@@ -115,37 +118,76 @@ def __init__(self, url):
         except IndexError:
             self.dkp = None
 
-    def get_data(self, force=False):
+    def call_api(self, url):
+        try:
+            print("updating via server...")
+            with urllib.request.urlopen(url) as response:
+                status_code = response.getcode()
+                content = response.read().decode('utf-8')
+                print("updated via server...")
+                return status_code, content
+        except urllib.error.URLError as e:
+            # If there's an error, use proxy
+            try:
+                print("updating via proxy...")
+                proxy_url = urljoin(self.PROXY, "proxy?url=%s/") % url
+                with urllib.request.urlopen(proxy_url) as proxy_response:
+                    proxy_status_code = proxy_response.getcode()
+                    proxy_content = proxy_response.read().decode('utf-8')
+                    print("updated via proxy...")
+                    return proxy_status_code, proxy_content
+            except urllib.error.URLError as proxy_error:
+                return None, f"An error occurred with both APIs: {e} and {proxy_error}"
+
+    def parse_result(self, api_response):
+        try:
+            # Parse the JSON response
+            json_response = json.loads(api_response[1])
+            return json_response
+        except json.JSONDecodeError as e:
+            return f"An error occurred while parsing JSON: {e}"
+
+    def get_data(self, force):
+        result = None
+        parsed_result = None
+
         if self.dkp is None:
             return
-
-        url = self.API_URL_NOT_FRESH % self.dkp
-        if force:
-            r = requests.get(url)
-        else:
-            r = request_with_cache(url)
 
-        if r.status_code != 200:
-            # fresh products have different api
+        url = self.API_URL_NOT_FRESH % self.dkp
+        api_response = self.call_api(url)
+        parsed_result = self.parse_result(api_response)
+        if int(parsed_result["status"]) == 200:
+            parsed_result = self.parse_result(api_response)
+        elif parsed_result["status"] == 302 and "fresh" in parsed_result["redirect_url"]["uri"]:
             url = self.API_URL_FRESH % self.dkp
-            if force:
-                r = requests.get(url)
+            api_response = self.call_api(url)
+            parsed_result = self.parse_result(api_response)
+            if parsed_result["status"] != 200:
+                print("Could not update!")
+                return
             else:
-                r = request_with_cache(url)
+                parsed_result = self.parse_result(api_response)
 
-            if r.status_code != 200:
-                return
+        else:
+            print("Could not update!")
+            print(url)
+            return
 
-        data = r.json()['data']
+        result = parsed_result["data"]
 
-        if data['product'].get('is_inactive'):
+
+        if result['product'].get('is_inactive'):
             return dict(cost='unavailable', img=None, title=None)
 
-        title = data['product']['title_fa']
-        if data['product']['status'] == 'marketable':
-            cost = int(data['product']['default_variant']['price']['rrp_price']) // 10
+        title = result['product']['title_fa']
+        if result['product']['status'] == 'marketable':
+            cost = int(result['product']['default_variant']['price']['rrp_price']) // 10
         else:
             cost = 'unavailable'
 
-        img = data['product']['images']['main']['url'][0]
+        img = result['product']['images']['main']['url'][0]
         return dict(cost=cost, img=img, title=title)
+
+
+

say/models/need_model.py

@@ -447,17 +447,22 @@ def update(self, force=False):
         from say.crawler import DigikalaCrawler
 
         if 'digikala' in self.link:
+            print("updating via crawler.....")
             data = DigikalaCrawler(self.link).get_data(force=force)
+            print("Done updating.")
         else:
             data = Crawler(self.link).get_data(force=force)
 
         if data is None:
+            print(f"Could not get data for: {self.id}")
             return
 
         img = data['img']
         title = data['title']
         cost = data['cost']
-
+        print(f"Id: {self.id}")
+        print(f"fetched title: {title}")
+        print(f"fetched cost: {cost}")
         if img:
             self.img = img
 

say/models/social_worker_model.py

@@ -203,18 +203,20 @@ def validate_password(self, password):
     def send_password(self, password):
         from say.authorization import create_sw_access_token
         from say.tasks import send_embeded_subject_email
-
-        send_embeded_subject_email.delay(
-            to=self.email,
-            html=render_template_i18n(
-                'social_worker_password.html',
-                social_worker=self,
-                surname=surname(self.gender),
-                password=password,
-                token=create_sw_access_token(self),
-                locale=self.locale,
-            ),
-        )
+        try:
+            send_embeded_subject_email.delay(
+                to=self.email,
+                html=render_template_i18n(
+                    'social_worker_password.html',
+                    social_worker=self,
+                    surname=surname(self.gender),
+                    password=password,
+                    token=create_sw_access_token(self),
+                    locale=self.locale,
+                ),
+            )
+        except Exception as e:
+            self.retry(exc=e, countdown=60)
 
     @staticmethod
     def generate_password():

say/tasks/update_needs.py

@@ -1,6 +1,6 @@
 from time import sleep
 
-from sqlalchemy import or_
+from sqlalchemy import or_, and_
 
 from say.celery import celery
 from say.orm import safe_commit
@@ -12,17 +12,19 @@ def update_needs(self):
 
     needs = self.session.query(Need).filter(
         Need.type == 1,
-        or_(
-            Need.status < 4,
-            Need.title.is_(None),
-        ),
+        Need.status < 3,
         Need.isDeleted.is_(False),
         Need.link.isnot(None),
     )
 
     t = []
+    counter = 0
+    print(f"Total needs to be updated: {needs.count()}")
     for need in needs:
+        counter+=1
         t.append(need.id)
+        print(f"{counter}/{needs.count()}-> updating need: {need.id}")
+        sleep(10)
         update_need.delay(need.id)
 
     return t
@@ -38,9 +40,7 @@ def update_needs(self):
 )
 def update_need(self, need_id, force=False):
     from say.models.need_model import Need
-
-    sleep(5)
-    need = self.session.query(Need).get(need_id)
+    need = self.session.query(Need).get(need_id)    
     data = need.update(force=force)
     safe_commit(self.session)
 

say/templates/en/base.html

@@ -140,7 +140,7 @@
         >
           <a href="http://say.company" target="_blank">
             <img
-              src="https://api.sayapp.company/public/resources/img/logo-gray.png"
+              src="https://api.sayapp.company/public/resources/img/the-logo.png"
               style="
                 height: 100%;
                 vertical-align: middle;

say/templates/en/cart_successful_payment.html

@@ -106,7 +106,7 @@
   <body>
     <div id="header">
       <div id="logoWrapper">
-        <img src="https://api.sayapp.company/public/resources/img/logo.png" />
+        <img src="https://api.sayapp.company/public/resources/img/the-logo.png" />
       </div>
       <p>What you hear is real</p>
       <p>Building a platform to share wealth and love, literally</p>

say/templates/en/successful_payment.html

@@ -106,7 +106,7 @@
   <body>
     <div id="header">
       <div id="logoWrapper">
-        <img src="https://api.sayapp.company/public/resources/img/logo.png" />
+        <img src="https://api.sayapp.company/public/resources/img/the-logo.png" />
       </div>
       <p>What you hear is real</p>
       <p>Building a platform to share wealth and love, literally</p>