From 7856e15cab0cf94a178826cd732ab4ecfd1740cb Mon Sep 17 00:00:00 2001
From: nully0x <40327060+nully0x@users.noreply.github.com>
Date: Wed, 12 Jul 2023 22:59:16 +0100
Subject: [PATCH] update sheet data script

---
 .../converter/push_data_to_ES.js              | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/csv-to-elastic-webhook/converter/push_data_to_ES.js b/csv-to-elastic-webhook/converter/push_data_to_ES.js
index 72fb7ec..06c9bca 100644
--- a/csv-to-elastic-webhook/converter/push_data_to_ES.js
+++ b/csv-to-elastic-webhook/converter/push_data_to_ES.js
@@ -2,9 +2,9 @@
 //To run copy the script to the sheet's appscript that needs to be ingested to ES.
 
 function postSheetDataToElasticsearch() {
-  const sheetName = 'sheet1'; 
-  const elasticsearchUrl = ''; //ES url from the specific index/indices
-  const apiKey = ' '; //your ES apiKey
+  const sheetName = 'sheet1';
+  const elasticsearchUrl = ' ';
+  const apiKey = ' ';
 
   const sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
   const dataRange = sheet.getDataRange();
@@ -16,7 +16,13 @@ function postSheetDataToElasticsearch() {
     const postData = {};
     headers.forEach((header, index) => {
       const value = row[index];
-      if (header === 'created_at') {
+      if (header === 'indexed_at') {
+        postData[header] = new Date().toISOString(); // Set the current date and time
+      } else if (header === 'Id') {
+        postData[header] = 'googlesheet-' + (rowIndex + 1); // Set the id column as "googlesheet-1", "googlesheet-2", ...
+      } else if (header === 'tags') {
+        postData[header] = [value.toLowerCase().trim()]; // Convert tags to lowercase
+      } else if (header === 'created_at') {
         if (value instanceof Date && !isNaN(value)) {
           postData[header] = value.toISOString();
         } else if (typeof value === 'string' && value.trim() !== '') {
@@ -30,7 +36,7 @@ function postSheetDataToElasticsearch() {
           postData[header] = new Date().toISOString(); // Assign current date if the value is empty or not a valid date
         }
       } else if (value !== '' && value !== null) {
-        if (header === 'authors' || header === 'categories' || header === 'tags') {
+        if (header === 'authors' || header === 'categories') {
           postData[header] = [value.trim()];
         } else {
           postData[header] = value;
@@ -41,6 +47,7 @@ function postSheetDataToElasticsearch() {
     const options = {
       method: 'post',
       contentType: 'application/json',
+      muteHttpExceptions: true,
       headers: {
         'Content-Type': 'application/json',
         'Authorization': 'ApiKey ' + apiKey
@@ -50,12 +57,14 @@ function postSheetDataToElasticsearch() {
 
     const response = UrlFetchApp.fetch(elasticsearchUrl, options);
     const responseCode = response.getResponseCode();
-    if (responseCode === 200) {
-      Logger.log('Could not send data to Elasticsearch (row ' + (rowIndex + 1) + ')');
-    } else {
+    if (responseCode >= 200 && responseCode !== 400) {
       Logger.log('Data sent to Elasticsearch (row ' + (rowIndex + 1) + '). Response code: ' + responseCode);
+    } else if (responseCode >= 400) {
+      Logger.log('Could not send data to Elasticsearch (row ' + (rowIndex + 1) + '). Response code: ' + responseCode);
+    } else {
+      Logger.log('Execution complete');
     }
 
-    // Utilities.sleep(); // Pause execution for specified seconds
+    // Utilities.sleep(); // Pause execution for 2 seconds
   });
 }