From 018d1ae0b3d940510c773a449ad60be0c3288a78 Mon Sep 17 00:00:00 2001
From: mgrani <michael.granitzer@uni-passau.de>
Date: Tue, 27 May 2025 09:45:19 +0200
Subject: [PATCH] fix: build script for opensearch indices improved.

---
 owilix/plugins/push/convert.py |  1 +
 scripts/ours_index.sh          | 48 +++++++++++++++++++++++++++++++---
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/owilix/plugins/push/convert.py b/owilix/plugins/push/convert.py
index fc678ba..b5a76a8 100644
--- a/owilix/plugins/push/convert.py
+++ b/owilix/plugins/push/convert.py
@@ -60,6 +60,7 @@ class OURSConverter:
             if json.get("id", None): # overwrite by internal id
                 json["_id"]=json["id"]
                 # todo: implement proper unique ids.
+            # todo: add main content converter and strip html.
             # creates unfortunately schema problems.
             #for f in ["json-ld", "microdata", "opengraph"]:
             #    try:
diff --git a/scripts/ours_index.sh b/scripts/ours_index.sh
index d7226eb..a6820cf 100755
--- a/scripts/ours_index.sh
+++ b/scripts/ours_index.sh
@@ -1,4 +1,46 @@
 #!/bin/bash
-#Script for creating a title index given the specifier
-source .env-rc # provide your own rc file for credentials
-owilix query less --remote $1 "select=url,id,title,warc_date,day,month,year,domain_label as ows_curlielabel" as_json=True | owilix plugin owilix.plugins.push.opensearch.OpenSearchIndexer index ows_ours_v2 convert_fn=owilix.plugins.push.convert.OURSConverter
\ No newline at end of file
+# Script for creating a title index given the specifier
+
+# Load environment variables from .env-rc
+source .env-rc
+
+# Debug: Print loaded environment variables
+echo "OWILIX_OPENSEARCH_USERNAME: $OWILIX_OPENSEARCH_USERNAME"
+echo "OWILIX_OPENSEARCH_HOST: $OWILIX_OPENSEARCH_HOST"
+
+# Check if required environment variables are set
+if [ -z "$OWILIX_OPENSEARCH_USERNAME" ] || [ -z "$OWILIX_OPENSEARCH_HOST" ]; then
+    echo "Error: Required environment variables are not set."
+    exit 1
+fi
+
+# Generate a unique name for the named pipe
+TIMESTAMP=$(date +%s%N)  # Current timestamp in nanoseconds
+RANDOM_NUM=$RANDOM        # Random number
+PIPE_NAME="owilix_pipe_${TIMESTAMP}_${RANDOM_NUM}"
+
+# Create a named pipe
+mkfifo "$PIPE_NAME"
+
+# Function to clean up on script exit
+cleanup() {
+    echo "Cleaning up..."
+    # Kill the background process
+    kill $QUERY_PID 2>/dev/null
+    # Remove the named pipe
+    rm -f "$PIPE_NAME"
+    exit
+}
+
+# Trap SIGINT (Ctrl+C) and call cleanup
+trap cleanup SIGINT
+
+# Run the query and write to the named pipe in the background
+owilix query less --remote "$1" "select=url,id,title,warc_date,day,month,year,keywords,author,curlielabels_en as ows_curlielabel" as_json=True json_file="$PIPE_NAME" &
+QUERY_PID=$!
+
+# Read from the named pipe and index using the plugin
+owilix plugin owilix.plugins.push.opensearch.OpenSearchIndexer index $2 convert_fn=owilix.plugins.push.convert.OURSConverter < "$PIPE_NAME"
+
+# Clean up after completion
+cleanup
-- 
2.51.2