From 0e6f5813ac39c16f0e5bcc1f2b080b53c0c3ecb2 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Wed, 20 Aug 2025 11:59:02 +0530 Subject: [PATCH 01/19] Added changes for meilisearch --- docker/build/meilisearch-devstack/Dockerfile | 18 +++++ docker/build/meilisearch/Dockerfile | 14 ++++ docker/plays/meilisearch.yml | 6 ++ playbooks/meilisearch.yml | 49 +++++++++++++ requirements/meilisearch.in | 5 ++ util/meilisearch/copy-index.sh | 50 +++++++++++++ .../meilisearch/forums-incremental-reindex.sh | 43 +++++++++++ util/meilisearch/requirements.txt | 18 +++++ util/meilisearch/verify-index-copy.py | 73 +++++++++++++++++++ vagrant/release/analyticstack/Vagrantfile | 3 +- 10 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 docker/build/meilisearch-devstack/Dockerfile create mode 100644 docker/build/meilisearch/Dockerfile create mode 100644 docker/plays/meilisearch.yml create mode 100644 playbooks/meilisearch.yml create mode 100644 requirements/meilisearch.in create mode 100644 util/meilisearch/copy-index.sh create mode 100644 util/meilisearch/forums-incremental-reindex.sh create mode 100644 util/meilisearch/requirements.txt create mode 100644 util/meilisearch/verify-index-copy.py diff --git a/docker/build/meilisearch-devstack/Dockerfile b/docker/build/meilisearch-devstack/Dockerfile new file mode 100644 index 000000000..c753f3ac2 --- /dev/null +++ b/docker/build/meilisearch-devstack/Dockerfile @@ -0,0 +1,18 @@ +# docker build -f docker/build/meilisearch-devstack/Dockerfile . -t edxops/meilisearch:devstack + +FROM getmeili/meilisearch:v1.10.0 +LABEL maintainer="edxops" + +# (Optional) Create a data directory +RUN mkdir -p /meili_data && chown -R root:root /meili_data + +# Set environment variables (can be overridden at runtime) +ENV MEILI_ENV=development \ + MEILI_HTTP_ADDR=0.0.0.0:7700 \ + MEILI_DB_PATH=/meili_data + +# Expose API port +EXPOSE 7700 + +# Default entrypoint +CMD ["meilisearch"] diff --git a/docker/build/meilisearch/Dockerfile b/docker/build/meilisearch/Dockerfile new file mode 100644 index 000000000..7ab72b4d7 --- /dev/null +++ b/docker/build/meilisearch/Dockerfile @@ -0,0 +1,14 @@ +FROM edxops/xenial-common:latest +LABEL maintainer="edxops" + +# Copy ansible roles/playbooks into container +ADD . /edx/app/edx_ansible/edx_ansible +WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays + +# Run ansible playbook for Meilisearch (you need meilisearch.yml role defined) +RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook meilisearch.yml -c local \ + -i '127.0.0.1,' + +WORKDIR /etc/meilisearch +CMD service meilisearch start && sleep 5 && tail -f /edx/var/log/meilisearch/meilisearch.log +EXPOSE 7700 diff --git a/docker/plays/meilisearch.yml b/docker/plays/meilisearch.yml new file mode 100644 index 000000000..3f37826cb --- /dev/null +++ b/docker/plays/meilisearch.yml @@ -0,0 +1,6 @@ +- hosts: all + become: True + roles: + - common + - oraclejdk + - meilisearch diff --git a/playbooks/meilisearch.yml b/playbooks/meilisearch.yml new file mode 100644 index 000000000..908c1618d --- /dev/null +++ b/playbooks/meilisearch.yml @@ -0,0 +1,49 @@ +- hosts: all + become: true + vars: + # By default take instances in and out of the elb(s) they + # are attached to + # To skip elb operations use "-e elb_pre_post=false" + elb_pre_post: true + # Number of instances to operate on at a time + serial_count: 1 + CLUSTER_NAME: "commoncluster" + MEILI_VERSION: "v1.11.0" # change to desired version + MEILI_PORT: 7700 + serial: "{{ serial_count }}" + pre_tasks: + - action: ec2_metadata_facts + when: elb_pre_post + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Instance De-register + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + region: us-east-1 + state: absent + wait_timeout: 60 + become: false + when: elb_pre_post + + roles: + - common + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - meilisearch # <--- replace elasticsearch with meilisearch + + post_tasks: + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Register instance in the elb + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + ec2_elbs: "{{ ec2_elbs }}" + region: us-east-1 + state: present + wait_timeout: 60 + become: false + when: elb_pre_post diff --git a/requirements/meilisearch.in b/requirements/meilisearch.in new file mode 100644 index 000000000..cd269ce12 --- /dev/null +++ b/requirements/meilisearch.in @@ -0,0 +1,5 @@ +# Requirements for util/meilisearch/verify-index-copy.py + +deepdiff==7.0.1 +jsonpickle==4.0.2 +meilisearch==0.31.0 \ No newline at end of file diff --git a/util/meilisearch/copy-index.sh b/util/meilisearch/copy-index.sh new file mode 100644 index 000000000..0d5bae14a --- /dev/null +++ b/util/meilisearch/copy-index.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./copy-index-meili.sh SOURCE_ES SERVER:PORT SOURCE_INDEX TARGET_MEILI_SERVER:PORT TARGET_INDEX +# +# Example: +# ./copy-index-meili.sh http://localhost:9200 source_index http://localhost:7700 target_index + +SOURCE_ES=$1 +SOURCE_INDEX=$2 +TARGET_MEILI=$3 +TARGET_INDEX=$4 +API_KEY="${5:-}" # Optional Meilisearch master key + +# Ensure target index exists +curl -s -X POST "$TARGET_MEILI/indexes" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d "{ \"uid\": \"$TARGET_INDEX\" }" || true + +# Scroll through Elasticsearch index +SCROLL="1m" +SIZE=1000 +SCROLL_ID=$(curl -s "$SOURCE_ES/$SOURCE_INDEX/_search?scroll=$SCROLL" \ + -H 'Content-Type: application/json' \ + -d "{ \"size\": $SIZE }" | jq -r '._scroll_id') + +while [ "$SCROLL_ID" != "null" ]; do + # Fetch batch from Elasticsearch + RESPONSE=$(curl -s "$SOURCE_ES/_search/scroll" \ + -H 'Content-Type: application/json' \ + -d "{ \"scroll\": \"$SCROLL\", \"scroll_id\": \"$SCROLL_ID\" }") + + DOCS=$(echo "$RESPONSE" | jq '.hits.hits[]._source') + COUNT=$(echo "$DOCS" | jq length) + + if [ "$COUNT" -eq 0 ]; then + break + fi + + # Push batch into Meilisearch + echo "$DOCS" | jq -s '.' | curl -s -X POST "$TARGET_MEILI/indexes/$TARGET_INDEX/documents" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + --data-binary @- + + # Next scroll id + SCROLL_ID=$(echo "$RESPONSE" | jq -r '._scroll_id') +done diff --git a/util/meilisearch/forums-incremental-reindex.sh b/util/meilisearch/forums-incremental-reindex.sh new file mode 100644 index 000000000..826d1c630 --- /dev/null +++ b/util/meilisearch/forums-incremental-reindex.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./incremental-reindex-meili.sh INDEX [WINDOW] [SLEEP_TIME] [BATCH_SIZE] +# +# Example: +# ./incremental-reindex-meili.sh content 30 + +INDEX="$1" +WINDOW="${2:-5}" +SLEEP_TIME="${3:-60}" +BATCH_SIZE="${4:-500}" +API_KEY="${MEILI_API_KEY:-}" +MEILI_URL="${MEILI_URL:-http://localhost:7700}" + +if [ "$SLEEP_TIME" -ge "$((WINDOW * 60))" ]; then + echo 'ERROR: SLEEP_TIME must not be longer than WINDOW, or else documents may be missed.' + exit 1 +fi + +while : ; do + echo "Fetching documents newer than $WINDOW minutes..." + + # Replace this with your actual data fetch logic (DB query or API) + # Here, we assume you have a script `fetch_new_docs.rb` that prints JSON array + NEW_DOCS=$(./fetch_new_docs.rb "$WINDOW" "$INDEX" "$BATCH_SIZE") + + if [ -n "$NEW_DOCS" ]; then + echo "$NEW_DOCS" | curl -s -X POST "$MEILI_URL/indexes/$INDEX/documents" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + --data-binary @- + echo "Indexed batch into $INDEX" + else + echo "No new docs." + fi + + echo "Sleeping $SLEEP_TIME seconds..." + sleep "$SLEEP_TIME" + + [ "$SLEEP_TIME" -le 0 ] && break +done diff --git a/util/meilisearch/requirements.txt b/util/meilisearch/requirements.txt new file mode 100644 index 000000000..8135aeeb4 --- /dev/null +++ b/util/meilisearch/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements/meilisearch.in +# +deepdiff==7.0.1 + # via -r requirements/meilisearch.in +jsonpickle==4.0.2 + # via -r requirements/meilisearch.in +meilisearch==0.31.0 + # via -r requirements/meilisearch.in +httpx==0.27.0 + # via meilisearch +anyio==4.4.0 + # via httpx +sniffio==1.3.1 + # via anyio \ No newline at end of file diff --git a/util/meilisearch/verify-index-copy.py b/util/meilisearch/verify-index-copy.py new file mode 100644 index 000000000..d32ff0da8 --- /dev/null +++ b/util/meilisearch/verify-index-copy.py @@ -0,0 +1,73 @@ +import random +import pprint +from deepdiff import DeepDiff +from meilisearch import Client + +def docs_match(old_doc, new_doc): + ignorable_fields = [] + diff = DeepDiff(old_doc, new_doc, ignore_order=True) + for field in ignorable_fields: + diff.pop(field, None) + return diff == {} + +def compare_indices(old_url, old_index, new_url, new_index, api_key=None, random_check_percentage=0.1): + old_client = Client(old_url, api_key) + new_client = Client(new_url, api_key) + + old_idx = old_client.index(old_index) + new_idx = new_client.index(new_index) + + # Compare stats + old_stats = old_idx.get_stats() + new_stats = new_idx.get_stats() + + old_count = old_stats['numberOfDocuments'] + new_count = new_stats['numberOfDocuments'] + + print("{}: Document count ({} = {})".format( + 'OK' if old_count == new_count else 'FAILURE', old_count, new_count + )) + + # Compare settings instead of mappings + old_settings = old_idx.get_settings() + new_settings = new_idx.get_settings() + diff = DeepDiff(old_settings, new_settings, ignore_order=True) + if diff != {}: + print("FAILURE: Index settings do not match") + pprint.pprint(diff) + else: + print("OK: Index settings match") + + # Random checks + sample_size = int(old_count * random_check_percentage) + if sample_size == 0: + print("Skipping random checks (too few docs)") + return + + print(f"Checking {sample_size} random documents...") + offset = 0 + limit = min(sample_size, 1000) # batch size + old_docs = old_idx.get_documents(limit=limit, offset=offset) + + checked = 0 + matching = 0 + for doc in random.sample(old_docs['results'], min(sample_size, len(old_docs['results']))): + doc_id = doc['id'] + new_doc = new_idx.get_document(doc_id) + if new_doc and docs_match(doc, new_doc): + matching += 1 + else: + print(f"FAILURE: Document with id {doc_id} does not match") + checked += 1 + + print("{}: Random documents matching ({} out of {}, {:.2f}%)".format( + 'OK' if matching == checked else 'FAILURE', matching, checked, (matching / checked * 100 if checked else 0) + )) + + +if __name__ == "__main__": + # Example usage: + # compare_indices("http://localhost:7700", "content_old", + # "http://localhost:7700", "content_new", + # api_key="masterKey", random_check_percentage=0.1) + pass diff --git a/vagrant/release/analyticstack/Vagrantfile b/vagrant/release/analyticstack/Vagrantfile index 7c5ed276c..37090d642 100644 --- a/vagrant/release/analyticstack/Vagrantfile +++ b/vagrant/release/analyticstack/Vagrantfile @@ -128,7 +128,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8002, host: 8002 # Ecommerce config.vm.network :forwarded_port, guest: 8120, host: 8120 # edX Notes Service config.vm.network :forwarded_port, guest: 8765, host: 8765 - config.vm.network :forwarded_port, guest: 9200, host: 9200 # Elasticsearch + # config.vm.network :forwarded_port, guest: 9200, host: 9200 # Elasticsearch + config.vm.network :forwarded_port, guest: 7700, host: 7700 # Meilisearch config.vm.network :forwarded_port, guest: 18080, host: 18080 # Forums config.vm.network :forwarded_port, guest: 8100, host: 8100 # Analytics Data API config.vm.network :forwarded_port, guest: 8110, host: 8110 # Insights From 9cdb6b9ed17aae2021e3a6d658369efcd222e800 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Wed, 3 Sep 2025 02:14:48 +0530 Subject: [PATCH 02/19] feat: added ansible roles for meilisearch and their related changes --- docker/build/edxapp/lms.yml | 6 +- docker/build/edxapp/studio.yml | 6 +- .../build/elasticsearch-devstack/Dockerfile | 10 +- docker/build/elasticsearch/Dockerfile | 20 ++-- docker/build/meilisearch-devstack/Dockerfile | 15 +-- docker/build/meilisearch/Dockerfile | 2 +- docker/plays/elasticsearch.yml | 12 +-- docker/plays/meilisearch.yml | 1 - playbooks/elasticsearch.yml | 90 +++++++++--------- playbooks/meilisearch.yml | 2 +- playbooks/roles/edxapp/defaults/main.yml | 14 ++- playbooks/roles/meilisearch/defaults/main.yml | 29 ++++++ playbooks/roles/meilisearch/meta/main.yml | 3 + playbooks/roles/meilisearch/tasks/main.yml | 94 +++++++++++++++++++ .../edx/etc/meilisearch/meilisearch.yml.j2 | 60 ++++++++++++ .../templates/etc/default/meilisearch.j2 | 50 ++++++++++ .../lib/systemd/system/meilisearch.service.j2 | 54 +++++++++++ requirements/meilisearch.in | 6 +- util/meilisearch/copy-index.sh | 40 ++++---- .../meilisearch/forums-incremental-reindex.sh | 13 ++- util/meilisearch/requirements.txt | 6 +- util/meilisearch/verify-index-copy.py | 72 +++++++++----- 22 files changed, 470 insertions(+), 135 deletions(-) create mode 100644 playbooks/roles/meilisearch/defaults/main.yml create mode 100644 playbooks/roles/meilisearch/meta/main.yml create mode 100644 playbooks/roles/meilisearch/tasks/main.yml create mode 100644 playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 create mode 100644 playbooks/roles/meilisearch/templates/etc/default/meilisearch.j2 create mode 100644 playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index c0e34bb4e..8f5971564 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -237,9 +237,9 @@ EDXNOTES_INTERNAL_API: http://edx.devstack.edx_notes_api:18120/api/v1 EDXNOTES_PUBLIC_API: http://localhost:18120/api/v1 EDX_API_KEY: PUT_YOUR_API_KEY_HERE EDX_PLATFORM_REVISION: master -ELASTIC_SEARCH_CONFIG: -- host: edx.devstack.elasticsearch710 - port: 9200 +MEILI_SEARCH_CONFIG: +- host: edx.devstack.meilisearch184 + port: 7700 use_ssl: false EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend EMAIL_HOST: localhost diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index d5603e0b4..5e7290616 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -214,9 +214,9 @@ ECOMMERCE_API_URL: http://localhost:8002/api/v2 ECOMMERCE_PUBLIC_URL_ROOT: http://localhost:8002 EDXMKTG_USER_INFO_COOKIE_NAME: edx-user-info EDX_PLATFORM_REVISION: master -ELASTIC_SEARCH_CONFIG: -- host: edx.devstack.elasticsearch710 - port: 9200 +MEILI_SEARCH_CONFIG: +- host: edx.devstack.meilisearch184 + port: 7700 use_ssl: false EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend EMAIL_HOST: localhost diff --git a/docker/build/elasticsearch-devstack/Dockerfile b/docker/build/elasticsearch-devstack/Dockerfile index b851031f0..0fe95b529 100644 --- a/docker/build/elasticsearch-devstack/Dockerfile +++ b/docker/build/elasticsearch-devstack/Dockerfile @@ -1,7 +1,7 @@ -# docker build -f docker/build/elasticsearch-devstack/Dockerfile . -t edxops/elasticsearch:devstack +# # docker build -f docker/build/elasticsearch-devstack/Dockerfile . -t edxops/elasticsearch:devstack -FROM elasticsearch:1.5.2 -LABEL maintainer="edxops" +# FROM elasticsearch:1.5.2 +# LABEL maintainer="edxops" -# Install the elastcisearch-head plugin (https://mobz.github.io/elasticsearch-head/) -RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head +# # Install the elastcisearch-head plugin (https://mobz.github.io/elasticsearch-head/) +# RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head diff --git a/docker/build/elasticsearch/Dockerfile b/docker/build/elasticsearch/Dockerfile index c869014ec..933383784 100644 --- a/docker/build/elasticsearch/Dockerfile +++ b/docker/build/elasticsearch/Dockerfile @@ -1,13 +1,13 @@ -FROM edxops/xenial-common:latest -LABEL maintainer="edxops" +# FROM edxops/xenial-common:latest +# LABEL maintainer="edxops" -ADD . /edx/app/edx_ansible/edx_ansible -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays +# ADD . /edx/app/edx_ansible/edx_ansible +# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -# Role is currently untagged -RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook elasticsearch.yml -c local \ - -i '127.0.0.1,' +# # Role is currently untagged +# RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook elasticsearch.yml -c local \ +# -i '127.0.0.1,' -WORKDIR /etc/elasticsearch -CMD service elasticsearch start && sleep 5 && tail -f /edx/var/log/elasticsearch/elasticsearch.log -EXPOSE 9200 9300 +# WORKDIR /etc/elasticsearch +# CMD service elasticsearch start && sleep 5 && tail -f /edx/var/log/elasticsearch/elasticsearch.log +# EXPOSE 9200 9300 diff --git a/docker/build/meilisearch-devstack/Dockerfile b/docker/build/meilisearch-devstack/Dockerfile index c753f3ac2..bffa1f7f9 100644 --- a/docker/build/meilisearch-devstack/Dockerfile +++ b/docker/build/meilisearch-devstack/Dockerfile @@ -3,16 +3,11 @@ FROM getmeili/meilisearch:v1.10.0 LABEL maintainer="edxops" -# (Optional) Create a data directory -RUN mkdir -p /meili_data && chown -R root:root /meili_data +# Default working directory +WORKDIR /meili_data -# Set environment variables (can be overridden at runtime) -ENV MEILI_ENV=development \ - MEILI_HTTP_ADDR=0.0.0.0:7700 \ - MEILI_DB_PATH=/meili_data - -# Expose API port +# Expose Meilisearch default port EXPOSE 7700 -# Default entrypoint -CMD ["meilisearch"] +# Start Meilisearch +CMD ["meilisearch", "--db-path", "/meili_data", "--http-addr", "0.0.0.0:7700"] diff --git a/docker/build/meilisearch/Dockerfile b/docker/build/meilisearch/Dockerfile index 7ab72b4d7..a251c6228 100644 --- a/docker/build/meilisearch/Dockerfile +++ b/docker/build/meilisearch/Dockerfile @@ -11,4 +11,4 @@ RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook meilisearch.yml WORKDIR /etc/meilisearch CMD service meilisearch start && sleep 5 && tail -f /edx/var/log/meilisearch/meilisearch.log -EXPOSE 7700 +EXPOSE 7700 7701 diff --git a/docker/plays/elasticsearch.yml b/docker/plays/elasticsearch.yml index 3783014eb..12278b143 100644 --- a/docker/plays/elasticsearch.yml +++ b/docker/plays/elasticsearch.yml @@ -1,6 +1,6 @@ -- hosts: all - become: True - roles: - - common - - oraclejdk - - elasticsearch +# - hosts: all +# become: True +# roles: +# - common +# - oraclejdk +# - elasticsearch diff --git a/docker/plays/meilisearch.yml b/docker/plays/meilisearch.yml index 3f37826cb..3c437f33a 100644 --- a/docker/plays/meilisearch.yml +++ b/docker/plays/meilisearch.yml @@ -2,5 +2,4 @@ become: True roles: - common - - oraclejdk - meilisearch diff --git a/playbooks/elasticsearch.yml b/playbooks/elasticsearch.yml index e5c3ff695..b5f1ebec1 100644 --- a/playbooks/elasticsearch.yml +++ b/playbooks/elasticsearch.yml @@ -1,45 +1,45 @@ -- hosts: all - become: True - vars: - # By default take instances in and out of the elb(s) they - # are attached to - # To skip elb operations use "-e elb_pre_post=fase" - elb_pre_post: true - # Number of instances to operate on at a time - serial_count: 1 - CLUSTER_NAME: "commoncluster" - serial: "{{ serial_count }}" - pre_tasks: - - action: ec2_metadata_facts - when: elb_pre_post - - debug: - var: ansible_ec2_instance_id - when: elb_pre_post - - name: Instance De-register - local_action: ec2_elb - args: - instance_id: "{{ ansible_ec2_instance_id }}" - region: us-east-1 - state: absent - wait_timeout: 60 - become: False - when: elb_pre_post - roles: - - common - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - elasticsearch - post_tasks: - - debug: - var: ansible_ec2_instance_id - when: elb_pre_post - - name: Register instance in the elb - local_action: ec2_elb - args: - instance_id: "{{ ansible_ec2_instance_id }}" - ec2_elbs: "{{ ec2_elbs }}" - region: us-east-1 - state: present - wait_timeout: 60 - become: False - when: elb_pre_post +# - hosts: all +# become: True +# vars: +# # By default take instances in and out of the elb(s) they +# # are attached to +# # To skip elb operations use "-e elb_pre_post=fase" +# elb_pre_post: true +# # Number of instances to operate on at a time +# serial_count: 1 +# CLUSTER_NAME: "commoncluster" +# serial: "{{ serial_count }}" +# pre_tasks: +# - action: ec2_metadata_facts +# when: elb_pre_post +# - debug: +# var: ansible_ec2_instance_id +# when: elb_pre_post +# - name: Instance De-register +# local_action: ec2_elb +# args: +# instance_id: "{{ ansible_ec2_instance_id }}" +# region: us-east-1 +# state: absent +# wait_timeout: 60 +# become: False +# when: elb_pre_post +# roles: +# - common +# - role: aws +# when: COMMON_ENABLE_AWS_ROLE +# - elasticsearch +# post_tasks: +# - debug: +# var: ansible_ec2_instance_id +# when: elb_pre_post +# - name: Register instance in the elb +# local_action: ec2_elb +# args: +# instance_id: "{{ ansible_ec2_instance_id }}" +# ec2_elbs: "{{ ec2_elbs }}" +# region: us-east-1 +# state: present +# wait_timeout: 60 +# become: False +# when: elb_pre_post diff --git a/playbooks/meilisearch.yml b/playbooks/meilisearch.yml index 908c1618d..f5c3bb463 100644 --- a/playbooks/meilisearch.yml +++ b/playbooks/meilisearch.yml @@ -31,7 +31,7 @@ - common - role: aws when: COMMON_ENABLE_AWS_ROLE - - meilisearch # <--- replace elasticsearch with meilisearch + - meilisearch # add the roles for meilisearch post_tasks: - debug: diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index bd923eee8..314e18a6f 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -165,10 +165,19 @@ EDXAPP_SEARCH_USE_SSL: false # list of dictionaries of the format # { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' } # http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch -EDXAPP_ELASTIC_SEARCH_CONFIG: +# EDXAPP_ELASTIC_SEARCH_CONFIG: +# - host: "{{ EDXAPP_SEARCH_HOST }}" +# port: "{{ EDXAPP_SEARCH_PORT }}" +# use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" + +# list of dictionaries of the format +# { 'url': 'http://hostname:port', 'api_key': 'masterKey' } +# https://github.com/meilisearch/meilisearch-python +EDXAPP_MEILI_SEARCH_CONFIG: - host: "{{ EDXAPP_SEARCH_HOST }}" port: "{{ EDXAPP_SEARCH_PORT }}" use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" + api_key: "{{ EDXAPP_SEARCH_API_KEY }}" # need to see how we can externalize this EDXAPP_SETTINGS: '{{ COMMON_EDXAPP_SETTINGS }}' @@ -1581,7 +1590,8 @@ generic_env_config: &edxapp_generic_env limits: "{{ EDXAPP_CODE_JAIL_LIMITS }}" user: '{{ edxapp_sandbox_user }}' AFFILIATE_COOKIE_NAME: "{{ EDXAPP_AFFILIATE_COOKIE_NAME }}" - ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" + # ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" + MEILI_SEARCH_CONFIG: "{{ EDXAPP_MEILI_SEARCH_CONFIG }}" PLATFORM_TWITTER_ACCOUNT: "{{ EDXAPP_PLATFORM_TWITTER_ACCOUNT }}" PLATFORM_FACEBOOK_ACCOUNT: "{{ EDXAPP_PLATFORM_FACEBOOK_ACCOUNT }}" HELP_TOKENS_BOOKS: "{{ EDXAPP_HELP_TOKENS_BOOKS }}" diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml new file mode 100644 index 000000000..248220a5d --- /dev/null +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -0,0 +1,29 @@ +--- +meilisearch_app_dir: "{{ COMMON_APP_DIR }}/meilisearch" +meilisearch_data_dir: "{{ COMMON_DATA_DIR }}/meilisearch" +meilisearch_log_dir: "{{ COMMON_LOG_DIR }}/meilisearch" +meilisearch_cfg_dir: "{{ COMMON_CFG_DIR }}/meilisearch" +# elasticsearch_apt_key_url: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" +# elasticsearch_repo: "deb [trusted=yes] https://artifacts.elastic.co/packages/7.x/apt stable main" +# elasticsearch_package_name: "elasticsearch" + +# Meilisearch doesn’t come from apt repo (like Elasticsearch), +# we fetch the binary directly from GitHub releases. +meilisearch_download_url: "https://github.com/meilisearch/meilisearch/releases/download/{{ meilisearch_version }}/meilisearch-linux-amd64" + +meilisearch_user: "meilisearch" +meilisearch_group: "meilisearch" + +# Defaults for a single server installation. +MEILISEARCH_CLUSTER_MEMBERS: [] # Not typically needed, but kept for parity +MEILISEARCH_ENV: "development" +# ELASTICSEARCH_HEAP_SIZE: "512m" +# Service management +MEILISEARCH_START_TIMEOUT: "300" +# Version of Meilisearch to install +meilisearch_version: "v1.10.0" + +# Network and security settings +meilisearch_host: "0.0.0.0" +meilisearch_port: 7700 +meilisearch_master_key: "" # Set if you want API key protection diff --git a/playbooks/roles/meilisearch/meta/main.yml b/playbooks/roles/meilisearch/meta/main.yml new file mode 100644 index 000000000..2083f0e12 --- /dev/null +++ b/playbooks/roles/meilisearch/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - common diff --git a/playbooks/roles/meilisearch/tasks/main.yml b/playbooks/roles/meilisearch/tasks/main.yml new file mode 100644 index 000000000..55a8293a5 --- /dev/null +++ b/playbooks/roles/meilisearch/tasks/main.yml @@ -0,0 +1,94 @@ +--- +# meilisearch +# +# Dependencies: +# +# * common +# +# Example play: +# +# This role installs and configures the Meilisearch service. +# It can be used for single-server installs (default). +# + +- name: create meilisearch group + group: + name: "{{ meilisearch_group }}" + state: present + tags: + - install + - install:base + +- name: create meilisearch user + user: + name: "{{ meilisearch_user }}" + group: "{{ meilisearch_group }}" + shell: /usr/sbin/nologin + system: yes + create_home: no + tags: + - install + - install:base + +- name: create directories + file: + path: "{{ item }}" + state: directory + owner: "{{ meilisearch_user }}" + group: "{{ meilisearch_group }}" + mode: 0755 + loop: + - "{{ meilisearch_data_dir }}" + - "{{ meilisearch_log_dir }}" + - "{{ meilisearch_cfg_dir }}" + tags: + - install + - install:base + +- name: download meilisearch binary + get_url: + url: "{{ meilisearch_download_url }}" + dest: "{{ meilisearch_app_dir }}/meilisearch" + mode: '0755' + tags: + - install + - install:base + register: meilisearch_download + +- name: drop the meilisearch config (env file) + template: + src: etc/default/meilisearch.j2 + dest: /etc/default/meilisearch + mode: 0644 + tags: + - install + - install:configuration + +- name: drop the meilisearch systemd service config + template: + src: lib/systemd/system/meilisearch.service.j2 + dest: "/lib/systemd/system/meilisearch.service" + mode: 0644 + tags: + - install + - install:configuration + +- name: Ensure meilisearch is enabled and started + service: + name: meilisearch + state: started + enabled: yes + tags: + - manage + - manage:start + +- name: Restart meilisearch when binary is updated + service: + name: meilisearch + state: restarted + enabled: yes + when: meilisearch_download.changed + tags: + - manage + - manage:restart + - install diff --git a/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 new file mode 100644 index 000000000..ab8fd1c47 --- /dev/null +++ b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 @@ -0,0 +1,60 @@ +# {{ ansible_managed }} + +# ======================== Meilisearch Configuration ========================= +# +# NOTE: Meilisearch comes with safe defaults for most settings. +# Only override what you need for your deployment. +# See official docs: +# https://www.meilisearch.com/docs/learn/configuration/instance_options +# +# This template generates a .env file that Meilisearch will load at startup. +# ---------------------------------------------------------------------------- + +# ---------------------------------- Paths ----------------------------------- +# Path to store Meilisearch data (indexes, snapshots, dumps) +MEILI_DB_PATH={{ meilisearch_data_dir }} + +# Path to store Meilisearch logs (stdout/stderr are default if unset) +MEILI_LOG_PATH={{ meilisearch_log_dir }} + +# ---------------------------------- Network --------------------------------- +# The IP address Meilisearch binds to (0.0.0.0 for all interfaces) +MEILI_HTTP_ADDR={{ meilisearch_bind_host | default("0.0.0.0") }} + +# The port Meilisearch listens on +MEILI_HTTP_PORT={{ meilisearch_port | default("7700") }} + +# ---------------------------------- Master Key ------------------------------- +# The master key is used to generate API keys. Comment out to disable auth. +# !! Make sure to secure this in production !! +MEILI_MASTER_KEY={{ meilisearch_master_key | default("") }} + +# ---------------------------------- Snapshots & Dumps ------------------------ +# Directory to store snapshots +MEILI_SNAPSHOT_DIR={{ meilisearch_snapshot_dir | default(meilisearch_data_dir + "/snapshots") }} + +# Directory to store dumps +MEILI_DUMP_DIR={{ meilisearch_dump_dir | default(meilisearch_data_dir + "/dumps") }} + +# Enable scheduled snapshots (true/false) +MEILI_SCHEDULE_SNAPSHOT={{ meilisearch_schedule_snapshot | default("false") }} + +# ---------------------------------- Clustering ------------------------------- +# Meilisearch does not support clustering (like Elasticsearch). +# Scaling is typically handled with sharding at the app level or proxies. +# This section is informational only. + +# ---------------------------------- Experimental ----------------------------- +# Maximum index size (bytes) before compaction +MEILI_MAX_INDEX_SIZE={{ meilisearch_max_index_size | default("10000000000") }} # 10GB default + +# Number of threads used by Meilisearch +MEILI_MAX_THREADS={{ meilisearch_max_threads | default(ansible_processor_vcpus | default("4")) }} + +# ---------------------------------- Security -------------------------------- +# Whether to disable analytics sent to Meilisearch (default: false) +MEILI_NO_ANALYTICS={{ meilisearch_no_analytics | default("true") }} + +# ---------------------------------- Various --------------------------------- +# Require explicit UID for created indexes (default: false) +MEILI_ENV={{ meilisearch_env | default("production") }} diff --git a/playbooks/roles/meilisearch/templates/etc/default/meilisearch.j2 b/playbooks/roles/meilisearch/templates/etc/default/meilisearch.j2 new file mode 100644 index 000000000..7536b2e84 --- /dev/null +++ b/playbooks/roles/meilisearch/templates/etc/default/meilisearch.j2 @@ -0,0 +1,50 @@ +# {{ ansible_managed }} + +################################ +# Meilisearch +################################ + +# Meilisearch binary directory +#MEILI_HOME=/usr/bin/meilisearch + +# Meilisearch configuration directory +MEILI_PATH_CONF={{ meilisearch_cfg_dir }} + +# Meilisearch data directory +MEILI_DB_PATH={{ meilisearch_data_dir }} + +# Meilisearch log directory +MEILI_LOG_PATH={{ meilisearch_log_dir }} + +# Meilisearch PID directory +#PID_DIR=/var/run/meilisearch + +# Additional Meilisearch options (passed as CLI args or env variables) +#MEILI_OPTS= + +# Configure restart on package upgrade (true, every other setting will lead to not restarting) +#RESTART_ON_UPGRADE=true + +################################ +# Meilisearch service +################################ + +# SysV init.d +# +# The number of seconds to wait before checking if Meilisearch started successfully as a daemon process +MEILI_STARTUP_SLEEP_TIME=5 + +################################ +# System properties +################################ + +# Specifies the maximum file descriptor number that can be opened by this process +# When using Systemd, this setting is ignored and the LimitNOFILE defined in +# /usr/lib/systemd/system/meilisearch.service takes precedence +#MAX_OPEN_FILES=65535 + +# The maximum number of bytes of memory that may be locked into RAM +# Set to "unlimited" if you want to avoid swapping issues. +# When using systemd, LimitMEMLOCK must be set in a unit file such as +# /etc/systemd/system/meilisearch.service.d/override.conf. +#MAX_LOCKED_MEMORY=unlimited diff --git a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 new file mode 100644 index 000000000..b978f6b28 --- /dev/null +++ b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 @@ -0,0 +1,54 @@ +[Unit] +Description=Meilisearch +Documentation=https://www.meilisearch.com +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +RuntimeDirectory=meilisearch +PrivateTmp=true +Environment=MEILI_HOME=/usr/bin/meilisearch +Environment=MEILI_PATH_CONF=/etc/meilisearch +Environment=PID_DIR=/var/run/meilisearch +EnvironmentFile=-/etc/default/meilisearch + +WorkingDirectory=/var/lib/meilisearch + +User={{ meilisearch_user }} +Group={{ meilisearch_group }} + +ExecStart=/usr/bin/meilisearch \ + --db-path {{ meilisearch_data_dir }} \ + --http-addr {{ meilisearch_bind_host }}:{{ meilisearch_bind_port }} \ + --env production \ + --master-key {{ meilisearch_master_key | default('changeme') }} \ + --log-level {{ meilisearch_log_level | default('INFO') }} + +StandardOutput=journal +StandardError=inherit + +# Resource limits (similar to Elasticsearch service) +LimitNOFILE=65535 +LimitNPROC=4096 +LimitAS=infinity +LimitFSIZE=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM is used to gracefully stop Meilisearch +KillSignal=SIGTERM +KillMode=process +SendSIGKILL=no + +# Meilisearch exits cleanly with code 0 +SuccessExitStatus=0 + +# Allow some startup time before systemd marks it failed +TimeoutStartSec={{ MEILISEARCH_START_TIMEOUT | default(30) }} + +[Install] +WantedBy=multi-user.target + +# Built for Meilisearch {{ meilisearch_version | default('latest') }} diff --git a/requirements/meilisearch.in b/requirements/meilisearch.in index cd269ce12..5e2d73291 100644 --- a/requirements/meilisearch.in +++ b/requirements/meilisearch.in @@ -1,5 +1,5 @@ # Requirements for util/meilisearch/verify-index-copy.py -deepdiff==7.0.1 -jsonpickle==4.0.2 -meilisearch==0.31.0 \ No newline at end of file +deepdiff==8.6.0 +meilisearch==0.37.0 +jsonpickle==4.0.2 \ No newline at end of file diff --git a/util/meilisearch/copy-index.sh b/util/meilisearch/copy-index.sh index 0d5bae14a..9d3ea1b67 100644 --- a/util/meilisearch/copy-index.sh +++ b/util/meilisearch/copy-index.sh @@ -2,49 +2,57 @@ set -euo pipefail # Usage: -# ./copy-index-meili.sh SOURCE_ES SERVER:PORT SOURCE_INDEX TARGET_MEILI_SERVER:PORT TARGET_INDEX -# -# Example: -# ./copy-index-meili.sh http://localhost:9200 source_index http://localhost:7700 target_index +# ./copy-index-meili.sh SOURCE_ES SOURCE_INDEX TARGET_MEILI TARGET_INDEX [API_KEY] SOURCE_ES=$1 SOURCE_INDEX=$2 TARGET_MEILI=$3 TARGET_INDEX=$4 -API_KEY="${5:-}" # Optional Meilisearch master key +API_KEY="${5:-}" -# Ensure target index exists +AUTH_HEADER=() +if [ -n "$API_KEY" ]; then + AUTH_HEADER=(-H "Authorization: Bearer $API_KEY") +fi + +# Recreate target index (optional but safer for fresh import) +curl -s -X DELETE "$TARGET_MEILI/indexes/$TARGET_INDEX" "${AUTH_HEADER[@]}" || true curl -s -X POST "$TARGET_MEILI/indexes" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer $API_KEY" \ - -d "{ \"uid\": \"$TARGET_INDEX\" }" || true + "${AUTH_HEADER[@]}" \ + -d "{ \"uid\": \"$TARGET_INDEX\" }" -# Scroll through Elasticsearch index +# Scroll through Elasticsearch SCROLL="1m" -SIZE=1000 +SIZE=500 SCROLL_ID=$(curl -s "$SOURCE_ES/$SOURCE_INDEX/_search?scroll=$SCROLL" \ -H 'Content-Type: application/json' \ -d "{ \"size\": $SIZE }" | jq -r '._scroll_id') while [ "$SCROLL_ID" != "null" ]; do - # Fetch batch from Elasticsearch RESPONSE=$(curl -s "$SOURCE_ES/_search/scroll" \ -H 'Content-Type: application/json' \ -d "{ \"scroll\": \"$SCROLL\", \"scroll_id\": \"$SCROLL_ID\" }") DOCS=$(echo "$RESPONSE" | jq '.hits.hits[]._source') - COUNT=$(echo "$DOCS" | jq length) + COUNT=$(echo "$DOCS" | jq -s 'length') if [ "$COUNT" -eq 0 ]; then break fi - # Push batch into Meilisearch - echo "$DOCS" | jq -s '.' | curl -s -X POST "$TARGET_MEILI/indexes/$TARGET_INDEX/documents" \ + echo "$DOCS" | jq -s '.' | curl -s -X POST \ + "$TARGET_MEILI/indexes/$TARGET_INDEX/documents" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer $API_KEY" \ + "${AUTH_HEADER[@]}" \ --data-binary @- - # Next scroll id SCROLL_ID=$(echo "$RESPONSE" | jq -r '._scroll_id') done + +# Cleanup scroll context +curl -s -X DELETE "$SOURCE_ES/_search/scroll" \ + -H 'Content-Type: application/json' \ + -d "{ \"scroll_id\": [\"$SCROLL_ID\"] }" > /dev/null || true + +echo "Copied index $SOURCE_INDEX from $SOURCE_ES to $TARGET_INDEX on $TARGET_MEILI" diff --git a/util/meilisearch/forums-incremental-reindex.sh b/util/meilisearch/forums-incremental-reindex.sh index 826d1c630..fc1c52cf2 100644 --- a/util/meilisearch/forums-incremental-reindex.sh +++ b/util/meilisearch/forums-incremental-reindex.sh @@ -19,19 +19,22 @@ if [ "$SLEEP_TIME" -ge "$((WINDOW * 60))" ]; then exit 1 fi +AUTH_HEADER=() +if [ -n "$API_KEY" ]; then + AUTH_HEADER=(-H "Authorization: Bearer $API_KEY") +fi + while : ; do echo "Fetching documents newer than $WINDOW minutes..." - # Replace this with your actual data fetch logic (DB query or API) - # Here, we assume you have a script `fetch_new_docs.rb` that prints JSON array NEW_DOCS=$(./fetch_new_docs.rb "$WINDOW" "$INDEX" "$BATCH_SIZE") - if [ -n "$NEW_DOCS" ]; then + if [ -n "$NEW_DOCS" ] && [ "$NEW_DOCS" != "[]" ]; then echo "$NEW_DOCS" | curl -s -X POST "$MEILI_URL/indexes/$INDEX/documents" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer $API_KEY" \ + "${AUTH_HEADER[@]}" \ --data-binary @- - echo "Indexed batch into $INDEX" + echo "Indexed $(echo "$NEW_DOCS" | jq length) docs into $INDEX" else echo "No new docs." fi diff --git a/util/meilisearch/requirements.txt b/util/meilisearch/requirements.txt index 8135aeeb4..aaf09d29e 100644 --- a/util/meilisearch/requirements.txt +++ b/util/meilisearch/requirements.txt @@ -4,15 +4,15 @@ # # pip-compile requirements/meilisearch.in # -deepdiff==7.0.1 +deepdiff==8.6.0 # via -r requirements/meilisearch.in jsonpickle==4.0.2 # via -r requirements/meilisearch.in -meilisearch==0.31.0 +meilisearch==0.37.0 # via -r requirements/meilisearch.in httpx==0.27.0 # via meilisearch anyio==4.4.0 # via httpx sniffio==1.3.1 - # via anyio \ No newline at end of file + # via anyio diff --git a/util/meilisearch/verify-index-copy.py b/util/meilisearch/verify-index-copy.py index d32ff0da8..1ddf35064 100644 --- a/util/meilisearch/verify-index-copy.py +++ b/util/meilisearch/verify-index-copy.py @@ -3,32 +3,52 @@ from deepdiff import DeepDiff from meilisearch import Client -def docs_match(old_doc, new_doc): - ignorable_fields = [] + +def docs_match(old_doc, new_doc, ignorable_fields=None): + """ + Compare two documents while ignoring order and optional ignorable fields. + """ + ignorable_fields = ignorable_fields or [] diff = DeepDiff(old_doc, new_doc, ignore_order=True) for field in ignorable_fields: diff.pop(field, None) return diff == {} -def compare_indices(old_url, old_index, new_url, new_index, api_key=None, random_check_percentage=0.1): + +def compare_indices( + old_url, + old_index, + new_url, + new_index, + api_key=None, + random_check_percentage=0.1, + ignorable_fields=None, +): + """ + Compare two Meilisearch indices by: + 1. Document counts + 2. Index settings + 3. Random sample of documents + """ + old_client = Client(old_url, api_key) new_client = Client(new_url, api_key) old_idx = old_client.index(old_index) new_idx = new_client.index(new_index) - # Compare stats + # ---- Compare stats ---- old_stats = old_idx.get_stats() new_stats = new_idx.get_stats() - old_count = old_stats['numberOfDocuments'] - new_count = new_stats['numberOfDocuments'] + old_count = old_stats["numberOfDocuments"] + new_count = new_stats["numberOfDocuments"] - print("{}: Document count ({} = {})".format( - 'OK' if old_count == new_count else 'FAILURE', old_count, new_count + print("{}: Document count (old={}, new={})".format( + "OK" if old_count == new_count else "FAILURE", old_count, new_count )) - # Compare settings instead of mappings + # ---- Compare settings ---- old_settings = old_idx.get_settings() new_settings = new_idx.get_settings() diff = DeepDiff(old_settings, new_settings, ignore_order=True) @@ -38,36 +58,46 @@ def compare_indices(old_url, old_index, new_url, new_index, api_key=None, random else: print("OK: Index settings match") - # Random checks + # ---- Random checks ---- sample_size = int(old_count * random_check_percentage) if sample_size == 0: print("Skipping random checks (too few docs)") return print(f"Checking {sample_size} random documents...") - offset = 0 - limit = min(sample_size, 1000) # batch size - old_docs = old_idx.get_documents(limit=limit, offset=offset) + + # Fetch docs from old index (cap batch size at 1000 for safety) + old_docs_response = old_idx.get_documents(limit=min(sample_size, 1000)) + old_docs = old_docs_response.get("results", []) checked = 0 matching = 0 - for doc in random.sample(old_docs['results'], min(sample_size, len(old_docs['results']))): - doc_id = doc['id'] - new_doc = new_idx.get_document(doc_id) - if new_doc and docs_match(doc, new_doc): + for doc in random.sample(old_docs, min(sample_size, len(old_docs))): + doc_id = doc["id"] + try: + new_doc = new_idx.get_document(doc_id) + except Exception: + new_doc = None + + if new_doc and docs_match(doc, new_doc, ignorable_fields): matching += 1 else: print(f"FAILURE: Document with id {doc_id} does not match") checked += 1 print("{}: Random documents matching ({} out of {}, {:.2f}%)".format( - 'OK' if matching == checked else 'FAILURE', matching, checked, (matching / checked * 100 if checked else 0) + "OK" if matching == checked else "FAILURE", + matching, + checked, + (matching / checked * 100 if checked else 0), )) if __name__ == "__main__": # Example usage: - # compare_indices("http://localhost:7700", "content_old", - # "http://localhost:7700", "content_new", - # api_key="masterKey", random_check_percentage=0.1) + # compare_indices( + # "http://localhost:7700", "content_old", + # "http://localhost:7700", "content_new", + # api_key="masterKey", random_check_percentage=0.1 + # ) pass From a14ddd114d52070930f373c6f3a17be7fd8c1b47 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 18 Sep 2025 21:52:22 +0530 Subject: [PATCH 03/19] fix: ansible roles changes for meilisearch --- playbooks/roles/meilisearch/defaults/main.yml | 2 ++ playbooks/roles/meilisearch/tasks/main.yml | 6 +++--- .../templates/lib/systemd/system/meilisearch.service.j2 | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index 248220a5d..4da3ea29e 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -25,5 +25,7 @@ meilisearch_version: "v1.10.0" # Network and security settings meilisearch_host: "0.0.0.0" +meilisearch_bind_host: "0.0.0.0" meilisearch_port: 7700 +meilisearch_bind_port: 7700 meilisearch_master_key: "" # Set if you want API key protection diff --git a/playbooks/roles/meilisearch/tasks/main.yml b/playbooks/roles/meilisearch/tasks/main.yml index 55a8293a5..5ee4a8120 100644 --- a/playbooks/roles/meilisearch/tasks/main.yml +++ b/playbooks/roles/meilisearch/tasks/main.yml @@ -48,7 +48,7 @@ - name: download meilisearch binary get_url: url: "{{ meilisearch_download_url }}" - dest: "{{ meilisearch_app_dir }}/meilisearch" + dest: "{{ meilisearch_app_dir }}" mode: '0755' tags: - install @@ -57,8 +57,8 @@ - name: drop the meilisearch config (env file) template: - src: etc/default/meilisearch.j2 - dest: /etc/default/meilisearch + src: edx/etc/meilisearch/meilisearch.j2 + dest: "{{ meilisearch_cfg_dir }}/meilisearch" mode: 0644 tags: - install diff --git a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 index b978f6b28..1d99abfb8 100644 --- a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 +++ b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 @@ -18,7 +18,7 @@ WorkingDirectory=/var/lib/meilisearch User={{ meilisearch_user }} Group={{ meilisearch_group }} -ExecStart=/usr/bin/meilisearch \ +ExecStart={{ meilisearch_app_dir }} \ --db-path {{ meilisearch_data_dir }} \ --http-addr {{ meilisearch_bind_host }}:{{ meilisearch_bind_port }} \ --env production \ From 70f61f4134a8e268618d89660a86292a3119c46c Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 23 Sep 2025 00:52:57 +0530 Subject: [PATCH 04/19] fix: externalize meili master key --- playbooks/roles/meilisearch/defaults/main.yml | 3 ++- playbooks/roles/meilisearch/tasks/main.yml | 2 +- .../templates/edx/etc/meilisearch/meilisearch.yml.j2 | 4 ++-- .../templates/lib/systemd/system/meilisearch.service.j2 | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index 4da3ea29e..dab4c3a74 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -28,4 +28,5 @@ meilisearch_host: "0.0.0.0" meilisearch_bind_host: "0.0.0.0" meilisearch_port: 7700 meilisearch_bind_port: 7700 -meilisearch_master_key: "" # Set if you want API key protection +MEILISEARCH_MASTER_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_MASTER_KEY', nested=True, region='us-east-1') }}" +MEILISEARCH_API_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_API_KEY', nested=True, region='us-east-1') }}" \ No newline at end of file diff --git a/playbooks/roles/meilisearch/tasks/main.yml b/playbooks/roles/meilisearch/tasks/main.yml index 5ee4a8120..ede6922d9 100644 --- a/playbooks/roles/meilisearch/tasks/main.yml +++ b/playbooks/roles/meilisearch/tasks/main.yml @@ -57,7 +57,7 @@ - name: drop the meilisearch config (env file) template: - src: edx/etc/meilisearch/meilisearch.j2 + src: edx/etc/meilisearch/meilisearch.yml.j2 dest: "{{ meilisearch_cfg_dir }}/meilisearch" mode: 0644 tags: diff --git a/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 index ab8fd1c47..d31b1eb31 100644 --- a/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 +++ b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 @@ -27,7 +27,7 @@ MEILI_HTTP_PORT={{ meilisearch_port | default("7700") }} # ---------------------------------- Master Key ------------------------------- # The master key is used to generate API keys. Comment out to disable auth. # !! Make sure to secure this in production !! -MEILI_MASTER_KEY={{ meilisearch_master_key | default("") }} +MEILISEARCH_MASTER_KEY={{ MEILISEARCH_MASTER_KEY }} # ---------------------------------- Snapshots & Dumps ------------------------ # Directory to store snapshots @@ -57,4 +57,4 @@ MEILI_NO_ANALYTICS={{ meilisearch_no_analytics | default("true") }} # ---------------------------------- Various --------------------------------- # Require explicit UID for created indexes (default: false) -MEILI_ENV={{ meilisearch_env | default("production") }} +MEILISEARCH_ENV={{ MEILISEARCH_ENV }} diff --git a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 index 1d99abfb8..469850780 100644 --- a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 +++ b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 @@ -21,8 +21,8 @@ Group={{ meilisearch_group }} ExecStart={{ meilisearch_app_dir }} \ --db-path {{ meilisearch_data_dir }} \ --http-addr {{ meilisearch_bind_host }}:{{ meilisearch_bind_port }} \ - --env production \ - --master-key {{ meilisearch_master_key | default('changeme') }} \ + --env {{ MEILISEARCH_ENV }} \ + --master-key {{ MEILISEARCH_MASTER_KEY }} \ --log-level {{ meilisearch_log_level | default('INFO') }} StandardOutput=journal From 815a1473232e02febaf2205902e2ef4d903182b3 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 23 Sep 2025 01:17:06 +0530 Subject: [PATCH 05/19] chore: updated port and address for meilisearch --- playbooks/roles/meilisearch/defaults/main.yml | 4 ++-- .../templates/edx/etc/meilisearch/meilisearch.yml.j2 | 2 +- .../templates/lib/systemd/system/meilisearch.service.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index dab4c3a74..cb59247af 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -25,8 +25,8 @@ meilisearch_version: "v1.10.0" # Network and security settings meilisearch_host: "0.0.0.0" -meilisearch_bind_host: "0.0.0.0" meilisearch_port: 7700 -meilisearch_bind_port: 7700 +MEILISEARCH_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" +MEILISEARCH_PUBLIC_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_MASTER_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_MASTER_KEY', nested=True, region='us-east-1') }}" MEILISEARCH_API_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_API_KEY', nested=True, region='us-east-1') }}" \ No newline at end of file diff --git a/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 index d31b1eb31..254dd5742 100644 --- a/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 +++ b/playbooks/roles/meilisearch/templates/edx/etc/meilisearch/meilisearch.yml.j2 @@ -19,7 +19,7 @@ MEILI_LOG_PATH={{ meilisearch_log_dir }} # ---------------------------------- Network --------------------------------- # The IP address Meilisearch binds to (0.0.0.0 for all interfaces) -MEILI_HTTP_ADDR={{ meilisearch_bind_host | default("0.0.0.0") }} +MEILI_HTTP_ADDR={{ meilisearch_host | default("0.0.0.0") }} # The port Meilisearch listens on MEILI_HTTP_PORT={{ meilisearch_port | default("7700") }} diff --git a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 index 469850780..6ea156ab9 100644 --- a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 +++ b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 @@ -20,7 +20,7 @@ Group={{ meilisearch_group }} ExecStart={{ meilisearch_app_dir }} \ --db-path {{ meilisearch_data_dir }} \ - --http-addr {{ meilisearch_bind_host }}:{{ meilisearch_bind_port }} \ + --http-addr {{ meilisearch_host }}:{{ meilisearch_port }} \ --env {{ MEILISEARCH_ENV }} \ --master-key {{ MEILISEARCH_MASTER_KEY }} \ --log-level {{ meilisearch_log_level | default('INFO') }} From 3bea7caa7ae71ae37d7be9e5a6c4551693956bc1 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 23 Sep 2025 18:15:05 +0530 Subject: [PATCH 06/19] fix: ssl handshake error --- playbooks/roles/common_vars/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 47685a49e..386e13901 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -57,6 +57,9 @@ COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple' COMMON_NPM_MIRROR_URL: 'https://registry.npmjs.org' COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=" +common_digicert_name: "DigiCertSHA2SecureServerCA.crt" +common_digicert_base_url: "https://cacerts.digicert.com" + COMMON_EDX_PPA: "deb http://ppa.edx.org {{ ansible_distribution_release }} main" COMMON_EDX_PPA_KEY_SERVER: "keyserver.ubuntu.com" COMMON_EDX_PPA_KEY_ID: "69464050" From 4a23cfd1aa0cfd04f20ec2f18a1ca7027f34eb43 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 23 Sep 2025 19:33:07 +0530 Subject: [PATCH 07/19] fix: change get_url with curl --- playbooks/roles/common/tasks/main.yml | 19 +++++++++++++++++++ playbooks/roles/meilisearch/defaults/main.yml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 4d4b840db..665a33875 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -13,6 +13,25 @@ path: "/usr/sbin/update-ca-certificates" register: update_ca_certificates +<<<<<<< HEAD +======= +# - name: Download digicert intermediate Certificate +# get_url: +# url: "{{ common_digicert_base_url }}/{{ common_digicert_name }}.pem" +# dest: "/usr/local/share/ca-certificates/{{ common_digicert_name }}" +# validate_certs: yes +# when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True + +- name: Download digicert intermediate Certificate with curl + shell: > + curl -fsSL {{ common_digicert_base_url }}/{{ common_digicert_name }}.pem + -o /usr/local/share/ca-certificates/{{ common_digicert_name }} + args: + warn: false + when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True + + +>>>>>>> 826d86bc3 (fix: change get_url with curl) - name: Update CA Certificates shell: /usr/sbin/update-ca-certificates when: update_ca_certificates is defined and update_ca_certificates.stat.exists == True diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index cb59247af..d017000c5 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -29,4 +29,4 @@ meilisearch_port: 7700 MEILISEARCH_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_PUBLIC_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_MASTER_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_MASTER_KEY', nested=True, region='us-east-1') }}" -MEILISEARCH_API_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_API_KEY', nested=True, region='us-east-1') }}" \ No newline at end of file +MEILISEARCH_API_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_API_KEY', nested=True, region='us-east-1') }}" From ef187ea284e7b79fd9ea11d7b457e3a0752d09ea Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 25 Sep 2025 17:01:31 +0530 Subject: [PATCH 08/19] fix: updated mongo role with task to wait for mongo to accept connection task --- playbooks/roles/mongo_5_0/tasks/main.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/playbooks/roles/mongo_5_0/tasks/main.yml b/playbooks/roles/mongo_5_0/tasks/main.yml index 10dd2484c..2e372aab9 100644 --- a/playbooks/roles/mongo_5_0/tasks/main.yml +++ b/playbooks/roles/mongo_5_0/tasks/main.yml @@ -80,6 +80,29 @@ - "install" - "install:app-configuration" +# Restart mongo service if configuration or packages changed +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# --- NEW TASK: Wait for MongoDB to accept connections --- +- name: wait for mongod to accept connections + wait_for: + host: 127.0.0.1 + port: 27017 + delay: 5 # initial wait + timeout: 60 # total timeout + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + # This will error when run on a new replica set, so we ignore_errors # and connect anonymously next. - name: determine if there is a replica set already From c3b3a9d573e2c5071172999df51d644b85ed6689 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 25 Sep 2025 17:33:58 +0530 Subject: [PATCH 09/19] fix: forcly adding tls in curl --- playbooks/roles/common/tasks/main.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 665a33875..1f19c5814 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -13,8 +13,6 @@ path: "/usr/sbin/update-ca-certificates" register: update_ca_certificates -<<<<<<< HEAD -======= # - name: Download digicert intermediate Certificate # get_url: # url: "{{ common_digicert_base_url }}/{{ common_digicert_name }}.pem" @@ -23,15 +21,14 @@ # when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True - name: Download digicert intermediate Certificate with curl - shell: > - curl -fsSL {{ common_digicert_base_url }}/{{ common_digicert_name }}.pem - -o /usr/local/share/ca-certificates/{{ common_digicert_name }} + shell: | + curl --tlsv1.2 -fsSL {{ common_digicert_base_url }}/{{ common_digicert_name }}.pem \ + -o /usr/local/share/ca-certificates/{{ common_digicert_name }} args: warn: false when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True - ->>>>>>> 826d86bc3 (fix: change get_url with curl) + - name: Update CA Certificates shell: /usr/sbin/update-ca-certificates when: update_ca_certificates is defined and update_ca_certificates.stat.exists == True From 10c754b872a65b966cba87a3b772fe6135b67894 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 25 Sep 2025 18:24:35 +0530 Subject: [PATCH 10/19] fix: updated the mongo 5 version task --- playbooks/roles/mongo_5_0/tasks/main.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/playbooks/roles/mongo_5_0/tasks/main.yml b/playbooks/roles/mongo_5_0/tasks/main.yml index 2e372aab9..520f1b0b3 100644 --- a/playbooks/roles/mongo_5_0/tasks/main.yml +++ b/playbooks/roles/mongo_5_0/tasks/main.yml @@ -81,15 +81,15 @@ - "install:app-configuration" # Restart mongo service if configuration or packages changed -- name: restart mongo service if we changed our configuration or upgraded mongo - service: - name: mongod - state: restarted - when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed - tags: - - "manage" - - "manage:start" - - "manage:db-replication" +# - name: restart mongo service if we changed our configuration or upgraded mongo +# service: +# name: mongod +# state: restarted +# when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed +# tags: +# - "manage" +# - "manage:start" +# - "manage:db-replication" # --- NEW TASK: Wait for MongoDB to accept connections --- - name: wait for mongod to accept connections From 9068f5962d7b52283437db57b1da56e13b9131e0 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Mon, 29 Sep 2025 16:22:40 +0530 Subject: [PATCH 11/19] chore: reverted changes in mongo 5 ansible role --- playbooks/roles/mongo_5_0/tasks/main.yml | 25 +----------------------- 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/playbooks/roles/mongo_5_0/tasks/main.yml b/playbooks/roles/mongo_5_0/tasks/main.yml index 520f1b0b3..f40ab2cef 100644 --- a/playbooks/roles/mongo_5_0/tasks/main.yml +++ b/playbooks/roles/mongo_5_0/tasks/main.yml @@ -80,29 +80,6 @@ - "install" - "install:app-configuration" -# Restart mongo service if configuration or packages changed -# - name: restart mongo service if we changed our configuration or upgraded mongo -# service: -# name: mongod -# state: restarted -# when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed -# tags: -# - "manage" -# - "manage:start" -# - "manage:db-replication" - -# --- NEW TASK: Wait for MongoDB to accept connections --- -- name: wait for mongod to accept connections - wait_for: - host: 127.0.0.1 - port: 27017 - delay: 5 # initial wait - timeout: 60 # total timeout - tags: - - "manage" - - "manage:start" - - "manage:db-replication" - # This will error when run on a new replica set, so we ignore_errors # and connect anonymously next. - name: determine if there is a replica set already @@ -424,4 +401,4 @@ when: MONGO_LOG_SERVERSTATUS tags: - "install" - - "install:app-configuration" + - "install:app-configuration" \ No newline at end of file From 708ea2f3f10417098194ee468856bd00bd79f6c2 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Mon, 29 Sep 2025 17:07:40 +0530 Subject: [PATCH 12/19] chore: reverted elastic search changes --- docker/build/edxapp/lms.yml | 3 +++ docker/build/edxapp/studio.yml | 3 +++ .../build/elasticsearch-devstack/Dockerfile | 10 +++++----- docker/build/elasticsearch/Dockerfile | 20 +++++++++---------- docker/plays/elasticsearch.yml | 12 +++++------ playbooks/roles/edxapp/defaults/main.yml | 12 +++++------ vagrant/release/analyticstack/Vagrantfile | 2 +- 7 files changed, 34 insertions(+), 28 deletions(-) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 8f5971564..339291182 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -237,6 +237,9 @@ EDXNOTES_INTERNAL_API: http://edx.devstack.edx_notes_api:18120/api/v1 EDXNOTES_PUBLIC_API: http://localhost:18120/api/v1 EDX_API_KEY: PUT_YOUR_API_KEY_HERE EDX_PLATFORM_REVISION: master +ELASTIC_SEARCH_CONFIG: +- host: edx.devstack.elasticsearch710 + port: 9200 MEILI_SEARCH_CONFIG: - host: edx.devstack.meilisearch184 port: 7700 diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 5e7290616..78af3066c 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -214,6 +214,9 @@ ECOMMERCE_API_URL: http://localhost:8002/api/v2 ECOMMERCE_PUBLIC_URL_ROOT: http://localhost:8002 EDXMKTG_USER_INFO_COOKIE_NAME: edx-user-info EDX_PLATFORM_REVISION: master +ELASTIC_SEARCH_CONFIG: +- host: edx.devstack.elasticsearch710 + port: 9200 MEILI_SEARCH_CONFIG: - host: edx.devstack.meilisearch184 port: 7700 diff --git a/docker/build/elasticsearch-devstack/Dockerfile b/docker/build/elasticsearch-devstack/Dockerfile index 0fe95b529..b851031f0 100644 --- a/docker/build/elasticsearch-devstack/Dockerfile +++ b/docker/build/elasticsearch-devstack/Dockerfile @@ -1,7 +1,7 @@ -# # docker build -f docker/build/elasticsearch-devstack/Dockerfile . -t edxops/elasticsearch:devstack +# docker build -f docker/build/elasticsearch-devstack/Dockerfile . -t edxops/elasticsearch:devstack -# FROM elasticsearch:1.5.2 -# LABEL maintainer="edxops" +FROM elasticsearch:1.5.2 +LABEL maintainer="edxops" -# # Install the elastcisearch-head plugin (https://mobz.github.io/elasticsearch-head/) -# RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head +# Install the elastcisearch-head plugin (https://mobz.github.io/elasticsearch-head/) +RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head diff --git a/docker/build/elasticsearch/Dockerfile b/docker/build/elasticsearch/Dockerfile index 933383784..c869014ec 100644 --- a/docker/build/elasticsearch/Dockerfile +++ b/docker/build/elasticsearch/Dockerfile @@ -1,13 +1,13 @@ -# FROM edxops/xenial-common:latest -# LABEL maintainer="edxops" +FROM edxops/xenial-common:latest +LABEL maintainer="edxops" -# ADD . /edx/app/edx_ansible/edx_ansible -# WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays +ADD . /edx/app/edx_ansible/edx_ansible +WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -# # Role is currently untagged -# RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook elasticsearch.yml -c local \ -# -i '127.0.0.1,' +# Role is currently untagged +RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook elasticsearch.yml -c local \ + -i '127.0.0.1,' -# WORKDIR /etc/elasticsearch -# CMD service elasticsearch start && sleep 5 && tail -f /edx/var/log/elasticsearch/elasticsearch.log -# EXPOSE 9200 9300 +WORKDIR /etc/elasticsearch +CMD service elasticsearch start && sleep 5 && tail -f /edx/var/log/elasticsearch/elasticsearch.log +EXPOSE 9200 9300 diff --git a/docker/plays/elasticsearch.yml b/docker/plays/elasticsearch.yml index 12278b143..3783014eb 100644 --- a/docker/plays/elasticsearch.yml +++ b/docker/plays/elasticsearch.yml @@ -1,6 +1,6 @@ -# - hosts: all -# become: True -# roles: -# - common -# - oraclejdk -# - elasticsearch +- hosts: all + become: True + roles: + - common + - oraclejdk + - elasticsearch diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 314e18a6f..95be79b66 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -165,10 +165,10 @@ EDXAPP_SEARCH_USE_SSL: false # list of dictionaries of the format # { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' } # http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch -# EDXAPP_ELASTIC_SEARCH_CONFIG: -# - host: "{{ EDXAPP_SEARCH_HOST }}" -# port: "{{ EDXAPP_SEARCH_PORT }}" -# use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" +EDXAPP_ELASTIC_SEARCH_CONFIG: + - host: "{{ EDXAPP_SEARCH_HOST }}" + port: "{{ EDXAPP_SEARCH_PORT }}" + use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" # list of dictionaries of the format # { 'url': 'http://hostname:port', 'api_key': 'masterKey' } @@ -177,7 +177,7 @@ EDXAPP_MEILI_SEARCH_CONFIG: - host: "{{ EDXAPP_SEARCH_HOST }}" port: "{{ EDXAPP_SEARCH_PORT }}" use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" - api_key: "{{ EDXAPP_SEARCH_API_KEY }}" # need to see how we can externalize this + api_key: "{{ EDXAPP_SEARCH_API_KEY }}" EDXAPP_SETTINGS: '{{ COMMON_EDXAPP_SETTINGS }}' @@ -1590,7 +1590,7 @@ generic_env_config: &edxapp_generic_env limits: "{{ EDXAPP_CODE_JAIL_LIMITS }}" user: '{{ edxapp_sandbox_user }}' AFFILIATE_COOKIE_NAME: "{{ EDXAPP_AFFILIATE_COOKIE_NAME }}" - # ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" + ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" MEILI_SEARCH_CONFIG: "{{ EDXAPP_MEILI_SEARCH_CONFIG }}" PLATFORM_TWITTER_ACCOUNT: "{{ EDXAPP_PLATFORM_TWITTER_ACCOUNT }}" PLATFORM_FACEBOOK_ACCOUNT: "{{ EDXAPP_PLATFORM_FACEBOOK_ACCOUNT }}" diff --git a/vagrant/release/analyticstack/Vagrantfile b/vagrant/release/analyticstack/Vagrantfile index 37090d642..2527e44bd 100644 --- a/vagrant/release/analyticstack/Vagrantfile +++ b/vagrant/release/analyticstack/Vagrantfile @@ -128,7 +128,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8002, host: 8002 # Ecommerce config.vm.network :forwarded_port, guest: 8120, host: 8120 # edX Notes Service config.vm.network :forwarded_port, guest: 8765, host: 8765 - # config.vm.network :forwarded_port, guest: 9200, host: 9200 # Elasticsearch + config.vm.network :forwarded_port, guest: 9200, host: 9200 # Elasticsearch config.vm.network :forwarded_port, guest: 7700, host: 7700 # Meilisearch config.vm.network :forwarded_port, guest: 18080, host: 18080 # Forums config.vm.network :forwarded_port, guest: 8100, host: 8100 # Analytics Data API From 7908350636888609410806a6b6b1d981bb66fba5 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 9 Oct 2025 11:13:02 +0530 Subject: [PATCH 13/19] chore: removed lookup for meili configuration --- playbooks/roles/meilisearch/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index d017000c5..e4a0bde89 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -28,5 +28,5 @@ meilisearch_host: "0.0.0.0" meilisearch_port: 7700 MEILISEARCH_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_PUBLIC_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" -MEILISEARCH_MASTER_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_MASTER_KEY', nested=True, region='us-east-1') }}" -MEILISEARCH_API_KEY: "{{ lookup('aws_secret', 'sandbox-secure/ansible/vars/developer-sandbox.MEILISEARCH_API_KEY', nested=True, region='us-east-1') }}" +MEILISEARCH_MASTER_KEY: "{{ MEILISEARCH_MASTER_KEY }}" +MEILISEARCH_API_KEY: "{{ MEILISEARCH_API_KEY }}" From 1fdc66b88e0df1a2e3d36910118d0abb0cd3b7b7 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Thu, 9 Oct 2025 13:40:26 +0530 Subject: [PATCH 14/19] chore: update the meili variables --- playbooks/roles/meilisearch/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index e4a0bde89..a40b1e3e5 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -28,5 +28,5 @@ meilisearch_host: "0.0.0.0" meilisearch_port: 7700 MEILISEARCH_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_PUBLIC_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" -MEILISEARCH_MASTER_KEY: "{{ MEILISEARCH_MASTER_KEY }}" -MEILISEARCH_API_KEY: "{{ MEILISEARCH_API_KEY }}" +MEILISEARCH_MASTER_KEY: "MEILISEARCH_MASTER_KEY" +MEILISEARCH_API_KEY: "MEILISEARCH_API_KEY" From d8ba338e3d0064aab837cc9410c277323b5d7f9e Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 14 Oct 2025 23:13:02 +0530 Subject: [PATCH 15/19] fix: updated meilisearch configuration --- playbooks/roles/meilisearch/defaults/main.yml | 10 ++++------ playbooks/roles/meilisearch/tasks/main.yml | 17 +++++++++++++---- .../lib/systemd/system/meilisearch.service.j2 | 7 ++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index a40b1e3e5..a40598fdb 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -3,9 +3,6 @@ meilisearch_app_dir: "{{ COMMON_APP_DIR }}/meilisearch" meilisearch_data_dir: "{{ COMMON_DATA_DIR }}/meilisearch" meilisearch_log_dir: "{{ COMMON_LOG_DIR }}/meilisearch" meilisearch_cfg_dir: "{{ COMMON_CFG_DIR }}/meilisearch" -# elasticsearch_apt_key_url: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" -# elasticsearch_repo: "deb [trusted=yes] https://artifacts.elastic.co/packages/7.x/apt stable main" -# elasticsearch_package_name: "elasticsearch" # Meilisearch doesn’t come from apt repo (like Elasticsearch), # we fetch the binary directly from GitHub releases. @@ -17,7 +14,6 @@ meilisearch_group: "meilisearch" # Defaults for a single server installation. MEILISEARCH_CLUSTER_MEMBERS: [] # Not typically needed, but kept for parity MEILISEARCH_ENV: "development" -# ELASTICSEARCH_HEAP_SIZE: "512m" # Service management MEILISEARCH_START_TIMEOUT: "300" # Version of Meilisearch to install @@ -28,5 +24,7 @@ meilisearch_host: "0.0.0.0" meilisearch_port: 7700 MEILISEARCH_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" MEILISEARCH_PUBLIC_URL: "http://{{ meilisearch_host }}:{{ meilisearch_port }}" -MEILISEARCH_MASTER_KEY: "MEILISEARCH_MASTER_KEY" -MEILISEARCH_API_KEY: "MEILISEARCH_API_KEY" + +# These are placeholders – actual secrets come from sandbox repo +MEILISEARCH_MASTER_KEY: "{{ lookup('env', 'MEILISEARCH_MASTER_KEY') | default('changeme', true) }}" +MEILISEARCH_API_KEY: "{{ lookup('env', 'MEILISEARCH_API_KEY') | default('changeme', true) }}" \ No newline at end of file diff --git a/playbooks/roles/meilisearch/tasks/main.yml b/playbooks/roles/meilisearch/tasks/main.yml index ede6922d9..9ebfd07b7 100644 --- a/playbooks/roles/meilisearch/tasks/main.yml +++ b/playbooks/roles/meilisearch/tasks/main.yml @@ -29,7 +29,6 @@ tags: - install - install:base - - name: create directories file: path: "{{ item }}" @@ -41,6 +40,7 @@ - "{{ meilisearch_data_dir }}" - "{{ meilisearch_log_dir }}" - "{{ meilisearch_cfg_dir }}" + - "{{ meilisearch_app_dir }}" tags: - install - install:base @@ -48,12 +48,21 @@ - name: download meilisearch binary get_url: url: "{{ meilisearch_download_url }}" - dest: "{{ meilisearch_app_dir }}" + dest: "{{ meilisearch_app_dir }}/meilisearch-linux-amd64" mode: '0755' + register: meilisearch_download + tags: + - install + - install:base + +- name: rename meilisearch binary to meilisearch + command: sudo mv "{{ meilisearch_app_dir }}/meilisearch-linux-amd64" "{{ meilisearch_app_dir }}/meilisearch" + args: + creates: "{{ meilisearch_app_dir }}/meilisearch" + when: meilisearch_download.changed tags: - install - install:base - register: meilisearch_download - name: drop the meilisearch config (env file) template: @@ -91,4 +100,4 @@ tags: - manage - manage:restart - - install + - install \ No newline at end of file diff --git a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 index 6ea156ab9..ea23bd8b7 100644 --- a/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 +++ b/playbooks/roles/meilisearch/templates/lib/systemd/system/meilisearch.service.j2 @@ -13,12 +13,13 @@ Environment=MEILI_PATH_CONF=/etc/meilisearch Environment=PID_DIR=/var/run/meilisearch EnvironmentFile=-/etc/default/meilisearch -WorkingDirectory=/var/lib/meilisearch +#WorkingDirectory=/var/lib/meilisearch +WorkingDirectory={{ meilisearch_data_dir }} User={{ meilisearch_user }} Group={{ meilisearch_group }} -ExecStart={{ meilisearch_app_dir }} \ +ExecStart={{ meilisearch_app_dir }}/meilisearch \ --db-path {{ meilisearch_data_dir }} \ --http-addr {{ meilisearch_host }}:{{ meilisearch_port }} \ --env {{ MEILISEARCH_ENV }} \ @@ -51,4 +52,4 @@ TimeoutStartSec={{ MEILISEARCH_START_TIMEOUT | default(30) }} [Install] WantedBy=multi-user.target -# Built for Meilisearch {{ meilisearch_version | default('latest') }} +# Built for Meilisearch {{ meilisearch_version | default('latest') }} \ No newline at end of file From 80dc57a8fc3d7bcbac5ec85c7229d9a9ac641053 Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Wed, 15 Oct 2025 12:56:25 +0530 Subject: [PATCH 16/19] feat: add meilisearch role --- playbooks/edx_continuous_integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index fdc34ee38..18cfab1e9 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -38,6 +38,7 @@ - oraclejdk - elasticsearch - opensearch + - meilisearch - forum - { role: "xqueue", update_users: True } - edx_ansible From d723048a666aefed7b8572a1e11cfd6cb342d596 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Mon, 3 Nov 2025 16:32:03 +0530 Subject: [PATCH 17/19] chore: updated meilisearch version --- playbooks/elasticsearch.yml | 90 +++++++++---------- playbooks/roles/meilisearch/defaults/main.yml | 2 +- playbooks/roles/meilisearch/tasks/main.yml | 2 +- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/playbooks/elasticsearch.yml b/playbooks/elasticsearch.yml index b5f1ebec1..19f258642 100644 --- a/playbooks/elasticsearch.yml +++ b/playbooks/elasticsearch.yml @@ -1,45 +1,45 @@ -# - hosts: all -# become: True -# vars: -# # By default take instances in and out of the elb(s) they -# # are attached to -# # To skip elb operations use "-e elb_pre_post=fase" -# elb_pre_post: true -# # Number of instances to operate on at a time -# serial_count: 1 -# CLUSTER_NAME: "commoncluster" -# serial: "{{ serial_count }}" -# pre_tasks: -# - action: ec2_metadata_facts -# when: elb_pre_post -# - debug: -# var: ansible_ec2_instance_id -# when: elb_pre_post -# - name: Instance De-register -# local_action: ec2_elb -# args: -# instance_id: "{{ ansible_ec2_instance_id }}" -# region: us-east-1 -# state: absent -# wait_timeout: 60 -# become: False -# when: elb_pre_post -# roles: -# - common -# - role: aws -# when: COMMON_ENABLE_AWS_ROLE -# - elasticsearch -# post_tasks: -# - debug: -# var: ansible_ec2_instance_id -# when: elb_pre_post -# - name: Register instance in the elb -# local_action: ec2_elb -# args: -# instance_id: "{{ ansible_ec2_instance_id }}" -# ec2_elbs: "{{ ec2_elbs }}" -# region: us-east-1 -# state: present -# wait_timeout: 60 -# become: False -# when: elb_pre_post +- hosts: all + become: True + vars: + # By default take instances in and out of the elb(s) they + # are attached to + # To skip elb operations use "-e elb_pre_post=fase" + elb_pre_post: true + # Number of instances to operate on at a time + serial_count: 1 + CLUSTER_NAME: "commoncluster" + serial: "{{ serial_count }}" + pre_tasks: + - action: ec2_metadata_facts + when: elb_pre_post + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Instance De-register + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + region: us-east-1 + state: absent + wait_timeout: 60 + become: False + when: elb_pre_post + roles: + - common + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - elasticsearch + post_tasks: + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Register instance in the elb + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + ec2_elbs: "{{ ec2_elbs }}" + region: us-east-1 + state: present + wait_timeout: 60 + become: False + when: elb_pre_post \ No newline at end of file diff --git a/playbooks/roles/meilisearch/defaults/main.yml b/playbooks/roles/meilisearch/defaults/main.yml index a40598fdb..d596b202a 100644 --- a/playbooks/roles/meilisearch/defaults/main.yml +++ b/playbooks/roles/meilisearch/defaults/main.yml @@ -17,7 +17,7 @@ MEILISEARCH_ENV: "development" # Service management MEILISEARCH_START_TIMEOUT: "300" # Version of Meilisearch to install -meilisearch_version: "v1.10.0" +meilisearch_version: "v1.11.0" # Network and security settings meilisearch_host: "0.0.0.0" diff --git a/playbooks/roles/meilisearch/tasks/main.yml b/playbooks/roles/meilisearch/tasks/main.yml index 9ebfd07b7..ea29e308a 100644 --- a/playbooks/roles/meilisearch/tasks/main.yml +++ b/playbooks/roles/meilisearch/tasks/main.yml @@ -56,7 +56,7 @@ - install:base - name: rename meilisearch binary to meilisearch - command: sudo mv "{{ meilisearch_app_dir }}/meilisearch-linux-amd64" "{{ meilisearch_app_dir }}/meilisearch" + command: mv "{{ meilisearch_app_dir }}/meilisearch-linux-amd64" "{{ meilisearch_app_dir }}/meilisearch" args: creates: "{{ meilisearch_app_dir }}/meilisearch" when: meilisearch_download.changed From 8dfefe36953f3b4643f5bb72320c46a906627cdd Mon Sep 17 00:00:00 2001 From: amits-2u Date: Tue, 4 Nov 2025 14:44:51 +0530 Subject: [PATCH 18/19] fix: resolving conflicts --- playbooks/roles/common_vars/defaults/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 386e13901..9ddb453a4 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -57,9 +57,6 @@ COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple' COMMON_NPM_MIRROR_URL: 'https://registry.npmjs.org' COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=" -common_digicert_name: "DigiCertSHA2SecureServerCA.crt" -common_digicert_base_url: "https://cacerts.digicert.com" - COMMON_EDX_PPA: "deb http://ppa.edx.org {{ ansible_distribution_release }} main" COMMON_EDX_PPA_KEY_SERVER: "keyserver.ubuntu.com" COMMON_EDX_PPA_KEY_ID: "69464050" @@ -307,4 +304,4 @@ CONFIGURE_JWTS: false COMMON_RETIREMENT_SERVICE_SETUP: false # How to log in as "lms" to xqueue. -COMMON_XQUEUE_LMS_PASSWORD: password +COMMON_XQUEUE_LMS_PASSWORD: password \ No newline at end of file From bfba9de921e15b2eb3c2856978687bd3d634d507 Mon Sep 17 00:00:00 2001 From: amits-2u Date: Mon, 17 Nov 2025 21:00:23 +0530 Subject: [PATCH 19/19] chore: updated as per copilot recommendation --- docker/build/meilisearch-devstack/Dockerfile | 2 +- playbooks/roles/common/tasks/main.yml | 18 +----------------- util/meilisearch/copy-index.sh | 2 +- util/meilisearch/forums-incremental-reindex.sh | 10 +++++++--- 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/docker/build/meilisearch-devstack/Dockerfile b/docker/build/meilisearch-devstack/Dockerfile index bffa1f7f9..8bcc3fd86 100644 --- a/docker/build/meilisearch-devstack/Dockerfile +++ b/docker/build/meilisearch-devstack/Dockerfile @@ -1,6 +1,6 @@ # docker build -f docker/build/meilisearch-devstack/Dockerfile . -t edxops/meilisearch:devstack -FROM getmeili/meilisearch:v1.10.0 +FROM getmeili/meilisearch:v1.11.0 LABEL maintainer="edxops" # Default working directory diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 1f19c5814..1586e2d66 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -13,22 +13,6 @@ path: "/usr/sbin/update-ca-certificates" register: update_ca_certificates -# - name: Download digicert intermediate Certificate -# get_url: -# url: "{{ common_digicert_base_url }}/{{ common_digicert_name }}.pem" -# dest: "/usr/local/share/ca-certificates/{{ common_digicert_name }}" -# validate_certs: yes -# when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True - -- name: Download digicert intermediate Certificate with curl - shell: | - curl --tlsv1.2 -fsSL {{ common_digicert_base_url }}/{{ common_digicert_name }}.pem \ - -o /usr/local/share/ca-certificates/{{ common_digicert_name }} - args: - warn: false - when: update_ca_certificates is defined and update_ca_certificates.results[0].stat.exists == True - - - name: Update CA Certificates shell: /usr/sbin/update-ca-certificates when: update_ca_certificates is defined and update_ca_certificates.stat.exists == True @@ -256,4 +240,4 @@ tags: - "install" - "install:configuration" - - "logrotate" + - "logrotate" \ No newline at end of file diff --git a/util/meilisearch/copy-index.sh b/util/meilisearch/copy-index.sh index 9d3ea1b67..57ee1ed77 100644 --- a/util/meilisearch/copy-index.sh +++ b/util/meilisearch/copy-index.sh @@ -2,7 +2,7 @@ set -euo pipefail # Usage: -# ./copy-index-meili.sh SOURCE_ES SOURCE_INDEX TARGET_MEILI TARGET_INDEX [API_KEY] +# ./copy-index.sh SOURCE_ES SOURCE_INDEX TARGET_MEILI TARGET_INDEX [API_KEY] SOURCE_ES=$1 SOURCE_INDEX=$2 diff --git a/util/meilisearch/forums-incremental-reindex.sh b/util/meilisearch/forums-incremental-reindex.sh index fc1c52cf2..63bfe376c 100644 --- a/util/meilisearch/forums-incremental-reindex.sh +++ b/util/meilisearch/forums-incremental-reindex.sh @@ -2,10 +2,10 @@ set -euo pipefail # Usage: -# ./incremental-reindex-meili.sh INDEX [WINDOW] [SLEEP_TIME] [BATCH_SIZE] +# ./forums-incremental-reindex.sh INDEX [WINDOW] [SLEEP_TIME] [BATCH_SIZE] # # Example: -# ./incremental-reindex-meili.sh content 30 +# ./forums-incremental-reindex.sh content 30 INDEX="$1" WINDOW="${2:-5}" @@ -27,7 +27,11 @@ fi while : ; do echo "Fetching documents newer than $WINDOW minutes..." - NEW_DOCS=$(./fetch_new_docs.rb "$WINDOW" "$INDEX" "$BATCH_SIZE") + # Fetch documents newer than $WINDOW minutes using Meilisearch API + SINCE_TIMESTAMP=$(date -u -d "$WINDOW minutes ago" +"%Y-%m-%dT%H:%M:%SZ") + NEW_DOCS=$(curl -s "${AUTH_HEADER[@]}" \ + "$MEILI_URL/indexes/$INDEX/documents?limit=$BATCH_SIZE" \ + | jq --arg since "$SINCE_TIMESTAMP" '[.[] | select(.updated_at >= $since)]') if [ -n "$NEW_DOCS" ] && [ "$NEW_DOCS" != "[]" ]; then echo "$NEW_DOCS" | curl -s -X POST "$MEILI_URL/indexes/$INDEX/documents" \