From 87786371bc20f8a9c7d74d14fd3827629b022d30 Mon Sep 17 00:00:00 2001 From: Oleksandr Glushchenko <117204585+LexGlu@users.noreply.github.com> Date: Mon, 15 May 2023 20:25:01 +0300 Subject: [PATCH 1/2] change elasticsearch to requests to redis/postgres --- db_dump/data_mapper.py | 2 +- docker-compose.yml | 14 +-------- ecom_website/settings.py | 9 +----- store/documents.py | 32 ------------------- store/templates/store/base.html | 2 +- store/urls.py | 4 +-- store/views/search.py | 55 +++++++++++++++++++++------------ 7 files changed, 41 insertions(+), 77 deletions(-) delete mode 100644 store/documents.py diff --git a/db_dump/data_mapper.py b/db_dump/data_mapper.py index 1f61361..710b68b 100644 --- a/db_dump/data_mapper.py +++ b/db_dump/data_mapper.py @@ -84,7 +84,7 @@ async def upload_image(image_url): f"VALUES ('{name}', '{description}', {digital}, {price}, {stock}, '{image}', {category_id});" ) - upload_image(data['product_img_url']) + asyncio.run(upload_image(data['product_img_url'])) print(f'Product {name} added to the database') # commit the changes to the database connection.commit() diff --git a/docker-compose.yml b/docker-compose.yml index 88a6e39..ba1baaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,21 +77,9 @@ services: volumes: - redis_data:/data - elasticsearch: - image: elasticsearch:7.17.10 # version 8 is not supported by django-elasticsearch-dsl - container_name: elasticsearch - restart: always - environment: - - discovery.type=single-node - ports: - - "9200:9200" - volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - volumes: postgres_data: static_volume: media_volume: db_dump: - redis_data: - elasticsearch_data: \ No newline at end of file + redis_data: \ No newline at end of file diff --git a/ecom_website/settings.py b/ecom_website/settings.py index 963669a..99e0ac5 100644 --- a/ecom_website/settings.py +++ b/ecom_website/settings.py @@ -48,7 +48,6 @@ 'phonenumber_field', 'rest_framework', 'rest_framework.authtoken', - 'django_elasticsearch_dsl', ] REST_FRAMEWORK = { @@ -175,10 +174,4 @@ EMAIL_PORT = 587 EMAIL_USE_TLS = True EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER') -EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') - -ELASTICSEARCH_DSL = { - 'default': { - 'hosts': os.environ.get('ELASTICSEARCH_HOST', 'elasticsearch:9200') - }, -} \ No newline at end of file +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') \ No newline at end of file diff --git a/store/documents.py b/store/documents.py deleted file mode 100644 index 1e7570d..0000000 --- a/store/documents.py +++ /dev/null @@ -1,32 +0,0 @@ -# define elastic search document mapping - -from django_elasticsearch_dsl import Document, fields -from django_elasticsearch_dsl.registries import registry - -from store.models import Product - -@registry.register_document -class ProductDocument(Document): - - category = fields.ObjectField(properties={ - 'name': fields.TextField(), - }) - - class Index: - # Name of the Elasticsearch index - name = 'products' - # See Elasticsearch Indices API reference for available settings - settings = {'number_of_shards': 1, - 'number_of_replicas': 0} - - class Django: - model = Product - - # The fields of the model you want to be indexed in Elasticsearch - fields = [ - 'id', - 'name', - 'description', - 'brand', - ] - \ No newline at end of file diff --git a/store/templates/store/base.html b/store/templates/store/base.html index b9bc089..c016cba 100644 --- a/store/templates/store/base.html +++ b/store/templates/store/base.html @@ -214,7 +214,7 @@ // Make an AJAX request to the search view const xhr = new XMLHttpRequest(); - xhr.open('GET', '{% url 'store:search_elastic' %}?query=' + query, true); + xhr.open('GET', '{% url 'store:search_rt' %}?query=' + query, true); xhr.onload = function () { if (xhr.status === 200) { diff --git a/store/urls.py b/store/urls.py index 2df3f0b..ddc0e83 100644 --- a/store/urls.py +++ b/store/urls.py @@ -3,7 +3,7 @@ from .views.category import CategoryView from .views.product import ProductView, add_review from .views.auth import sign_up, log_in, log_out, update_cart_after_logout -from .views.search import search_product, search_elastic +from .views.search import search_product, search_rt from .views.cart import cart, checkout, update_item, process_order from .views.customer_views import customer_orders @@ -16,7 +16,7 @@ path('login/', log_in, name='login'), path('logout/', log_out, name='logout'), path('search/', search_product, name='search'), - path('search_elastic/', search_elastic, name='search_elastic'), + path('search_rt/', search_rt, name='search_rt'), path('cart/', cart, name='cart'), path('checkout/', checkout, name='checkout'), path('update_item/', update_item, name='update_item'), diff --git a/store/views/search.py b/store/views/search.py index 4e3b533..de50f9a 100644 --- a/store/views/search.py +++ b/store/views/search.py @@ -3,9 +3,9 @@ from django.db.models import Q from django.core.paginator import Paginator from django.core.cache import cache -from store.documents import ProductDocument from django.http import JsonResponse -from elasticsearch_dsl import Q as DSLQ +from functools import reduce +from operator import and_ def search_product(request): query = request.GET.get('query') @@ -31,28 +31,43 @@ def search_product(request): return render(request, 'store/search.html', context) -def search_elastic(request): +def search_rt(request): query = request.GET.get('query', '').lower() + q_string = f'?{query}' if not query: return JsonResponse([], safe=False) + + keywords = query.split() + + + # AJAX request to postgreSQL database + + products = cache.get(q_string) + + if not products: + products = cache.get('all_products') + if not products: + products = Product.objects.all() + cache.set('all_products', products) + + filters = [ + Q(name__icontains=keyword) | + Q(description__icontains=keyword) | + Q(category__name__icontains=keyword) + for keyword in keywords + ] + + products = products.filter(reduce(and_, filters)).distinct() + cache.set(q_string, products[:5]) - # Split the query string into individual words - words = query.split() - - # Construct a list of wildcard queries for each word in the query - wildcard_queries = [ - DSLQ('wildcard', name='*' + word + '*') | - DSLQ('wildcard', description='*' + word + '*') | - DSLQ('wildcard', brand='*' + word + '*') - for word in words - ] - - # Perform a search query on the Elasticsearch index where name, description, or brand fields contain all of the words - products = ProductDocument.search().query( - DSLQ('bool', must=wildcard_queries) - ) - - results = [product.to_dict() for product in products[0:5]] + # Serialize products to a JSON serializable format + results = [] + for product in products[:5]: + serialized_product = { + 'id': product.pk, + 'name': product.name, + } + results.append(serialized_product) return JsonResponse(results, safe=False) \ No newline at end of file From 9695cdf7612164133925518aaa3fafcc245528c2 Mon Sep 17 00:00:00 2001 From: Oleksandr Glushchenko <117204585+LexGlu@users.noreply.github.com> Date: Tue, 16 May 2023 11:10:19 +0300 Subject: [PATCH 2/2] fix cached results --- store/views/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/views/search.py b/store/views/search.py index de50f9a..f95165c 100644 --- a/store/views/search.py +++ b/store/views/search.py @@ -33,11 +33,11 @@ def search_product(request): def search_rt(request): query = request.GET.get('query', '').lower() - q_string = f'?{query}' if not query: return JsonResponse([], safe=False) + q_string = f'?rt-{query}' keywords = query.split()