From 92cdda5a8bdf060a88fdb34a083abfb49008852c Mon Sep 17 00:00:00 2001 From: Juan Mrad Date: Wed, 22 Jul 2026 12:31:08 -0500 Subject: [PATCH] [Package] replace unidecode with ISC anyascii (#442) --- .github/workflows/license-check-python.yml | 3 +- .../src/osprey/engine/stdlib/udfs/string.py | 4 +- .../engine/stdlib/udfs/tests/test_strings.py | 20 ++++++++ pyproject.toml | 2 +- uv.lock | 46 +++++++++---------- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/.github/workflows/license-check-python.yml b/.github/workflows/license-check-python.yml index c058c1e..cb8a7c1 100644 --- a/.github/workflows/license-check-python.yml +++ b/.github/workflows/license-check-python.yml @@ -47,7 +47,6 @@ jobs: # First-party packages (no license metadata needed): # osprey-rpc, osprey-worker, example_plugins, example_atproto_plugins # Packages pending license policy review: - # unidecode: GPL v2+ (#354) # tld: GPL v2 / LGPL / MPL 1.1 (#355) # psycopg2-binary: LGPL (#357) # simplejson: Academic Free License + MIT (#358) @@ -57,4 +56,4 @@ jobs: google-crc32c \ ddtrace \ osprey-rpc osprey-worker example_plugins example_atproto_plugins \ - unidecode tld psycopg2-binary simplejson text-unidecode + tld psycopg2-binary simplejson text-unidecode diff --git a/osprey_worker/src/osprey/engine/stdlib/udfs/string.py b/osprey_worker/src/osprey/engine/stdlib/udfs/string.py index 8c6eafb..3844a7a 100644 --- a/osprey_worker/src/osprey/engine/stdlib/udfs/string.py +++ b/osprey_worker/src/osprey/engine/stdlib/udfs/string.py @@ -8,6 +8,7 @@ from itertools import chain from typing import Literal, Set, cast from urllib.parse import ParseResult, urlparse, urlunparse +from anyascii import anyascii from osprey.engine.stdlib.udfs._prelude import ( ArgumentsBase, ConstExpr, @@ -15,7 +16,6 @@ from osprey.engine.stdlib.udfs._prelude import ( UDFBase, ValidationContext, ) -from unidecode import unidecode from .categories import UdfCategories @@ -396,7 +396,7 @@ class StringClean(UDFBase[StringCleaningArguments, str]): s = new_s if arguments.unidecode: - s = unidecode(s) + s = anyascii(s) if arguments.upper and not arguments.lower: s = s.upper() diff --git a/osprey_worker/src/osprey/engine/stdlib/udfs/tests/test_strings.py b/osprey_worker/src/osprey/engine/stdlib/udfs/tests/test_strings.py index 1cffdcc..878e983 100644 --- a/osprey_worker/src/osprey/engine/stdlib/udfs/tests/test_strings.py +++ b/osprey_worker/src/osprey/engine/stdlib/udfs/tests/test_strings.py @@ -271,6 +271,26 @@ def test_string_normalization(s: Scenario, execute: ExecuteFunction) -> None: assert e not in result +@pytest.mark.parametrize( + 's,expected', + [ + ('Κνωσός', 'Knwsos'), # Greek -> Latin transliteration (ω -> w per Greek romanization) + ('René', 'Rene'), # accented Latin + ('日本語', 'RiBenYu'), # CJK characters (anyascii does not insert spaces between characters) + ('Москва', 'Moskva'), # Cyrillic + ('hello', 'hello'), # pure ASCII passes through unchanged + ('café', 'cafe'), # common accented word + ], +) +def test_string_clean_unidecode(s: str, expected: str, execute: ExecuteFunction) -> None: + data = execute( + f""" + S = StringClean(s="{s}", unidecode=True) + """ + ) + assert data['S'] == expected + + @pytest.mark.parametrize( 'text,expected_result', [ diff --git a/pyproject.toml b/pyproject.toml index 68b9acd..b028006 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ common = [ "traitlets==5.14.3", "typing-extensions==4.12.2", "typing-inspect==0.9.0", - "unidecode==1.3.8", + "anyascii==0.3.3", "werkzeug==1.0.1", # --- Type stubs (used by mypy, not imported directly) --- "types-cachetools==6.1.0.20250717", diff --git a/uv.lock b/uv.lock index 6f3f6d8..f17f1dc 100644 --- a/uv.lock +++ b/uv.lock @@ -26,6 +26,7 @@ overrides = [{ name = "ddtrace", specifier = ">=2.17.2,<3.0.0" }] [manifest.dependency-groups] common = [ + { name = "anyascii", specifier = "==0.3.3" }, { name = "blinker", specifier = "==1.9.0" }, { name = "click", specifier = "==7.1.2" }, { name = "datadog", specifier = "==0.51.0" }, @@ -109,7 +110,6 @@ common = [ { name = "types-werkzeug", specifier = "==1.0.9" }, { name = "typing-extensions", specifier = "==4.12.2" }, { name = "typing-inspect", specifier = "==0.9.0" }, - { name = "unidecode", specifier = "==1.3.8" }, { name = "werkzeug", specifier = "==1.0.1" }, ] dev = [ @@ -152,6 +152,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/70/72e4ab117425ccdc4d10bd523a94c1baa051a15586057d64a4c6888f9e3f/aiodns-4.0.4-py3-none-any.whl", hash = "sha256:c24dd605bac70a1676ce503f967a98483ff163507198557d8e9db16267e6cfd2", size = 12696, upload-time = "2026-05-20T01:54:14.134Z" }, ] +[[package]] +name = "anyascii" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/ba/edebda727008390936da4a9bf677c19cd63b32d51e864656d2cbd1028e25/anyascii-0.3.3.tar.gz", hash = "sha256:c94e9dd9d47b3d9494eca305fef9447d00b4bf1a32aff85aa746fa3ec7fb95c3", size = 264680, upload-time = "2025-06-29T03:33:30.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/76/783b75a21ce3563b8709050de030ae253853b147bd52e141edc1025aa268/anyascii-0.3.3-py3-none-any.whl", hash = "sha256:f5ab5e53c8781a36b5a40e1296a0eeda2f48c649ef10c3921c1381b1d00dee7a", size = 345090, upload-time = "2025-06-29T03:33:28.356Z" }, +] + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -882,7 +891,7 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine == 'x86_64'", ] dependencies = [ - { name = "six", marker = "platform_machine == 'x86_64'" }, + { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/1c/c42834d4fee45c5cf2d9546e97e879a1cbcdecfd16eb1a12144dcb91edae/grpcio-1.49.1.tar.gz", hash = "sha256:d4725fc9ec8e8822906ae26bb26f5546891aa7fbc3443de970cc556d43a5c99f", size = 22059239, upload-time = "2022-09-22T03:02:44.376Z" } wheels = [ @@ -956,9 +965,9 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine == 'x86_64'", ] dependencies = [ - { name = "googleapis-common-protos", marker = "platform_machine == 'x86_64'" }, - { name = "grpcio", version = "1.49.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64'" }, - { name = "protobuf", marker = "platform_machine == 'x86_64'" }, + { name = "googleapis-common-protos" }, + { name = "grpcio", version = "1.49.1", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/99/7cd6af0c6da3084fa39818af3e3de1f4ca21127004e87f39a2affe290521/grpcio-status-1.49.1.tar.gz", hash = "sha256:658f48dc146ee0c7b6eebd302d74e0d45c00727c20035ff51d68750dbaccf5d9", size = 13505, upload-time = "2022-09-22T03:02:56.074Z" } wheels = [ @@ -977,9 +986,9 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine != 'x86_64'", ] dependencies = [ - { name = "googleapis-common-protos", marker = "platform_machine != 'x86_64'" }, - { name = "grpcio", version = "1.53.2", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64'" }, - { name = "protobuf", marker = "platform_machine != 'x86_64'" }, + { name = "googleapis-common-protos" }, + { name = "grpcio", version = "1.53.2", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, ] sdist = { url = "https://files.pythonhosted.org/packages/45/6c/ed089e2b0d9215a1405de8790b2d6593780690d2732e0c208da27dfd86c4/grpcio-status-1.53.2.tar.gz", hash = "sha256:9a0823b99218424f012f59f6e9d17e8f569a67f0ca47cbca27dbf3bdf72c2248", size = 13505, upload-time = "2023-08-02T16:55:34.192Z" } wheels = [ @@ -998,9 +1007,9 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine == 'x86_64'", ] dependencies = [ - { name = "grpcio", version = "1.49.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64'" }, - { name = "protobuf", marker = "platform_machine == 'x86_64'" }, - { name = "setuptools", marker = "platform_machine == 'x86_64'" }, + { name = "grpcio", version = "1.49.1", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/e4/3416d25aebc4477141a491fae2c9494c5e0437a706c59103a936aac7d072/grpcio-tools-1.49.1.tar.gz", hash = "sha256:84cc64e5b46bad43d5d7bd2fd772b656eba0366961187a847e908e2cb735db91", size = 2252679, upload-time = "2022-09-22T03:03:00.279Z" } wheels = [ @@ -1022,9 +1031,9 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine != 'x86_64'", ] dependencies = [ - { name = "grpcio", version = "1.53.2", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64'" }, - { name = "protobuf", marker = "platform_machine != 'x86_64'" }, - { name = "setuptools", marker = "platform_machine != 'x86_64'" }, + { name = "grpcio", version = "1.53.2", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/4a/d176f59e4b080f8bcd1242ca9bfa9d1a77a9ef1620725eef846f112b8a99/grpcio-tools-1.53.2.tar.gz", hash = "sha256:1b10f95d1f4af2b9d63c9beccf386b3a5e1a92f14b3f64a9a8385b6ba091e0dc", size = 2257313, upload-time = "2023-08-02T16:55:36.975Z" } wheels = [ @@ -2694,15 +2703,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, ] -[[package]] -name = "unidecode" -version = "1.3.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/89/19151076a006b9ac0dd37b1354e031f5297891ee507eb624755e58e10d3e/Unidecode-1.3.8.tar.gz", hash = "sha256:cfdb349d46ed3873ece4586b96aa75258726e2fa8ec21d6f00a591d98806c2f4", size = 192701, upload-time = "2024-01-11T11:58:35.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/b7/6ec57841fb67c98f52fc8e4a2d96df60059637cba077edc569a302a8ffc7/Unidecode-1.3.8-py3-none-any.whl", hash = "sha256:d130a61ce6696f8148a3bd8fe779c99adeb4b870584eeb9526584e9aa091fd39", size = 235494, upload-time = "2024-01-11T11:58:33.012Z" }, -] - [[package]] name = "urllib3" version = "2.7.0" -- 2.51.2