diff --git a/owilix/core/duckdb.py b/owilix/core/duckdb.py index 6cd874a..fdc2948 100644 --- a/owilix/core/duckdb.py +++ b/owilix/core/duckdb.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from functools import wraps from string import Template from typing import Dict, Generator, List, Callable, Tuple +from uuid import uuid4 import duckdb from fsspec import AbstractFileSystem @@ -548,7 +549,6 @@ class OWIDuckDBCopy (OWIDuckDBSelect): # Parameters for chunk processing offset = 0 - file_counter = 0 # Determine the total number of rows in owi_slice total_rows = self.retry_then_raise(conn, r"SELECT COUNT(*) FROM owi_slice").fetchone()[0] _sub_path = pq_batch.query_args.get("sub_path","") @@ -557,7 +557,7 @@ class OWIDuckDBCopy (OWIDuckDBSelect): # Define the output file path for each chunk output_file_path = os.path.join(self.output_dir,_sub_path) if not os.path.exists(output_file_path): os.makedirs(output_file_path) - output_file_path = os.path.join(output_file_path, f"metadata_{file_counter}.parquet") + output_file_path = os.path.join(output_file_path, f"metadata_{str(uuid4())}.parquet") # Extract and write the chunk directly to a file, excluding the source_file column copy_query = f""" COPY ( @@ -571,7 +571,6 @@ class OWIDuckDBCopy (OWIDuckDBSelect): self.retry_then_raise(conn, copy_query) # Increment counters for the next chunk offset += self.chunk_size - file_counter += 1 # Final query to count distinct source files and total rows count_query = f"""