From 47f05b73cc524e43a1e1e73cbc68b298e530c9d1 Mon Sep 17 00:00:00 2001 From: semantic-release-mgrani Date: Tue, 19 Nov 2024 16:26:54 +0100 Subject: [PATCH] 0.15.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 8 ++++ owilix/_version.py | 2 +- pyproject.toml | 2 +- scripts/owi-os-index.sh | 96 ----------------------------------------- sql/ours_mappings.json | 0 5 files changed, 10 insertions(+), 98 deletions(-) delete mode 100755 scripts/owi-os-index.sh create mode 100644 sql/ours_mappings.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0b98d..514e36c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # CHANGELOG +## v0.15.0 (2024-11-19) + +### Features + +- Opensearch plug plus converter alpha version + ([`9779dfd`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9779dfd4c5b7e834180f3257f9c64f0e77e473eb)) + + ## v0.14.0 (2024-11-14) ### Features diff --git a/owilix/_version.py b/owilix/_version.py index 6b4a99b..b18d451 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # These version placeholders will be replaced later during substitution. -__version__ = "0.14.0-post.1+004060a" +__version__ = "0.15.0" __version_tuple__ = (0, 14, 0, "post", 1, "004060a") diff --git a/pyproject.toml b/pyproject.toml index 2e73713..375de00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "owilix" -version = "0.14.0" +version = "0.15.0" description = "OWILIX - the Command Line Interface for slicing and consuming the Open Web Index. " authors = ["Michael Granitzer "] license = "MIT" diff --git a/scripts/owi-os-index.sh b/scripts/owi-os-index.sh deleted file mode 100755 index e6ab2c3..0000000 --- a/scripts/owi-os-index.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Script: owi-os-index -# Usage: owi-os-index / [username] [password] [batch-size] -# Description: This script reads JSON from stdin, batches it, and sends the batches to an OpenSearch cluster for push. -# It can create the index if it does not exist. - -# Default batch size (can be overridden by passing as a fifth argument) -DEFAULT_BATCH_SIZE=2 - -# Parse command-line arguments -ENDPOINT_URL=$1 -USERNAME=${2:-$OPENSEARCH_USERNAME} -PASSWORD=${3:-$OPENSEARCH_PASSWORD} -BATCH_SIZE=${4:-$DEFAULT_BATCH_SIZE} - -# Check for required parameters -if [[ -z "$ENDPOINT_URL" || -z "$USERNAME" ]]; then - echo "Usage: $0 / [username] [password] [batch-size]" - exit 1 -fi - -# Function to create the index if it doesn't exist -create_index_if_not_exists() { - STATUS=$(curl -s -k -o /dev/null -w "%{http_code}" -u "$USERNAME:$PASSWORD" "$ENDPOINT_URL") - if [[ "$STATUS" -eq 000 ]]; then - echo "Error: Unable to connect to $ENDPOINT_URL" - exit 1 - elif [[ "$STATUS" -eq 404 ]]; then - echo "Index does not exist. Creating index..." - - # JSON payload to create the index - JSON_PAYLOAD='{ - "settings": { "index": { "number_of_shards": 1, "number_of_replicas": 1 } } - }' - - # Execute the curl command to create the index - curl -X PUT "$ENDPOINT_URL" \ - -u "$USERNAME:$PASSWORD" \ - -H 'Content-Type: application/json' \ - -d "$JSON_PAYLOAD" - - echo "Index created." - else - echo "Index exists or connection failed. HTTP Status: $STATUS" - fi -} - -# Initialize variables for batch processing -batch=() -count=0 -total_count=0 - -# Call the function to create the index if necessary -create_index_if_not_exists - -# Read JSON input from stdin, line by line -while IFS= read -r line; do - # Append the JSON line to the batch - batch+=("$line") - count=$((count + 1)) - total_count=$((total_count + 1)) - - # Display the running total count - echo -ne "\rProcessing count: $total_count" - - # When batch size is reached, send the batch to OpenSearch - if (( count >= BATCH_SIZE )); then - # Prepare bulk data with proper format and final newline - bulk_data="" - for json in "${batch[@]}"; do - bulk_data+="{ \"index\" : {} }\n$json\n" - done - - # Send batch to OpenSearch with final newline - echo -e "$bulk_data" | curl -s -k -u "$USERNAME:$PASSWORD" -H "Content-Type: application/json" -X POST "$ENDPOINT_URL/_bulk" --data-binary @- - - # Reset the batch - batch=() - count=0 - fi -done - -# Send any remaining data in the batch -if (( count > 0 )); then - bulk_data="" - for json in "${batch[@]}"; do - bulk_data+="{ \"index\" : {} }\n$json\n" - done - - # Send the final batch with trailing newline - echo -e "$bulk_data" | curl -s -k -u "$USERNAME:$PASSWORD" -H "Content-Type: application/json" -X POST "$ENDPOINT_URL/_bulk" --data-binary @- -fi - -# Move to the next line after finishing -echo -e "\nData indexing completed. Total records processed: $total_count." diff --git a/sql/ours_mappings.json b/sql/ours_mappings.json new file mode 100644 index 0000000..e69de29 -- 2.51.2