Configure tokenization for keyword search

import weaviate
from weaviate.classes.config import Property, DataType, Tokenization, Configure

import weaviate from 'weaviate-client';

// Instantiate your client (not shown). e.g.:
// const client = await weaviate.connectToWeaviateCloud(...) or
// const client = await weaviate.connectToLocal();

// START CustomStopwordsCreate
import io.weaviate.client6.v1.api.WeaviateClient;
import io.weaviate.client6.v1.api.collections.Property;
import io.weaviate.client6.v1.api.collections.TextAnalyzer;
import io.weaviate.client6.v1.api.collections.Tokenization;
import io.weaviate.client6.v1.api.collections.VectorConfig;

await client.Collections.Create(
    new CollectionCreateParams
    {
        Name = "AccentFoldingDemo",
        Properties =
        [
            new Property
            {
                Name = "text_default",
                DataType = DataType.Text,
                PropertyTokenization = PropertyTokenization.Word,
            },
            new Property
            {
                Name = "text_folded",
                DataType = DataType.Text,
                PropertyTokenization = PropertyTokenization.Word,
                TextAnalyzer = new TextAnalyzerConfig
                {
                    AsciiFold = new AsciiFoldConfig(),
                },
            },
            new Property
            {
                Name = "text_folded_keep_e",
                DataType = DataType.Text,
                PropertyTokenization = PropertyTokenization.Word,
                TextAnalyzer = new TextAnalyzerConfig
                {
                    AsciiFold = new AsciiFoldConfig(Ignore: ["é"]),
                },
            },
        ],
        VectorConfig = new VectorConfigList
        {
            Configure.Vector("default", v => v.SelfProvided()),
        },
    }
);

products = client.collections.get("AccentFoldingDemo")

test_strings = [
    "Café Crème Bio",
    "Łódź Ceramics",
    "São Paulo Sandals",
    "Müller Bräu",
]

for text in test_strings:
    products.data.insert(
        properties={
            "text_default": text,
            "text_folded": text,
            "text_folded_keep_e": text,
        }
    )

const products = client.collections.use('AccentFoldingDemo');

const testStrings = ['Café Crème Bio', 'Łódź Ceramics', 'São Paulo Sandals', 'Müller Bräu'];

for (const text of testStrings) {
  await products.data.insert({
    text_default: text,
    text_folded: text,
    text_folded_keep_e: text,
  });
}

CollectionHandle<Map<String, Object>> products =
    client.collections.use(COLLECTION);

List<String> testStrings = List.of(
    "Café Crème Bio",
    "Łódź Ceramics",
    "São Paulo Sandals",
    "Müller Bräu");

for (String text : testStrings) {
  products.data.insert(Map.of(
      "text_default", text,
      "text_folded", text,
      "text_folded_keep_e", text));
}

var products = client.Collections.Use("AccentFoldingDemo");

string[] testStrings =
[
    "Café Crème Bio",
    "Łódź Ceramics",
    "São Paulo Sandals",
    "Müller Bräu",
];

foreach (var text in testStrings)
{
    await products.Data.Insert(
        new
        {
            text_default = text,
            text_folded = text,
            text_folded_keep_e = text,
        }
    );
}

from weaviate.classes.query import Filter

queries = ["cafe", "Café", "lodz", "sao paulo", "muller"]
properties = ["text_default", "text_folded", "text_folded_keep_e"]

for query in queries:
    print(f'\nQuery: "{query}"')
    for prop in properties:
        response = products.query.fetch_objects(
            filters=Filter.by_property(prop).equal(query),
        )
        matches = [o.properties[prop] for o in response.objects]
        print(f"  {prop}: {matches if matches else 'no match'}")

import { Filters } from 'weaviate-client';

const queries = ['cafe', 'Café', 'lodz', 'sao paulo', 'muller'];
const properties = ['text_default', 'text_folded', 'text_folded_keep_e'];

for (const query of queries) {
  console.log(`\nQuery: "${query}"`);
  for (const prop of properties) {
    const response = await products.query.fetchObjects({
      filters: products.filter.byProperty(prop).equal(query),
    });
    const matches = response.objects.map((o) => o.properties[prop] as string);
    console.log(`  ${prop}: ${matches.length ? JSON.stringify(matches) : 'no match'}`);
  }
}

String[] queries = {"cafe", "Café", "lodz", "sao paulo", "muller"};
String[] properties = {"text_default", "text_folded", "text_folded_keep_e"};

for (String query : queries) {
  System.out.println("\nQuery: \"" + query + "\"");
  for (String prop : properties) {
    var response = products.query.fetchObjects(
        q -> q.filters(Filter.property(prop).eq(query)));
    var matches = response.objects().stream()
        .map(o -> (String) o.properties().get(prop))
        .toList();
    System.out.println(
        "  " + prop + ": " + (matches.isEmpty() ? "no match" : matches));
  }
}

string[] queries = ["cafe", "Café", "lodz", "sao paulo", "muller"];
string[] properties = ["text_default", "text_folded", "text_folded_keep_e"];

foreach (var query in queries)
{
    Console.WriteLine($"\nQuery: \"{query}\"");
    foreach (var prop in properties)
    {
        var response = await products.Query.FetchObjects(
            filters: Filter.Property(prop).IsEqual(query)
        );
        var matches = response
            .Objects.Select(o => (string)o.Properties[prop])
            .ToList();
        Console.WriteLine(
            $"  {prop}: {(matches.Count == 0 ? "no match" : string.Join(", ", matches))}"
        );
    }
}

import weaviate
from weaviate.classes.config import Property, DataType, Tokenization, Configure

import weaviate from 'weaviate-client';

// Instantiate your client (not shown). e.g.:
// const client = await weaviate.connectToWeaviateCloud(...) or
// const client = await weaviate.connectToLocal();

import io.weaviate.client6.v1.api.WeaviateClient;
import io.weaviate.client6.v1.api.collections.Property;
import io.weaviate.client6.v1.api.collections.TextAnalyzer;
import io.weaviate.client6.v1.api.collections.Tokenization;
import io.weaviate.client6.v1.api.collections.VectorConfig;
// END AccentFoldingCreateCollection

var presets = new Dictionary<string, IList<string>>
{
    ["fr"] = ["le", "la", "les", "un", "une", "des", "du", "de", "et"],
};

await client.Collections.Create(
    new CollectionCreateParams
    {
        Name = "StopwordsDemo",
        InvertedIndexConfig = new InvertedIndexConfig { StopwordPresets = presets },
        Properties =
        [
            new Property
            {
                Name = "name_en",
                DataType = DataType.Text,
                PropertyTokenization = PropertyTokenization.Word,
                TextAnalyzer = new TextAnalyzerConfig { StopwordPreset = "en" },
            },
            new Property
            {
                Name = "name_fr",
                DataType = DataType.Text,
                PropertyTokenization = PropertyTokenization.Word,
                TextAnalyzer = new TextAnalyzerConfig { StopwordPreset = "fr" },
            },
        ],
        VectorConfig = new VectorConfigList
        {
            Configure.Vector("default", v => v.SelfProvided()),
        },
    }
);

products = client.collections.get("StopwordsDemo")

products.data.insert_many([
    {
        "name_en": "The Blue Cup and the Bowl",
        "name_fr": "La Tasse Bleue et le Bol",
    },
    {
        "name_en": "A Red Plate with the Saucer",
        "name_fr": "Une Assiette Rouge avec la Soucoupe",
    },
])

const products = client.collections.use('StopwordsDemo');

await products.data.insertMany([
  {
    name_en: 'The Blue Cup and the Bowl',
    name_fr: 'La Tasse Bleue et le Bol',
  },
  {
    name_en: 'A Red Plate with the Saucer',
    name_fr: 'Une Assiette Rouge avec la Soucoupe',
  },
]);

CollectionHandle<Map<String, Object>> products =
    client.collections.use(COLLECTION);

products.data.insert(Map.of(
    "name_en", "The Blue Cup and the Bowl",
    "name_fr", "La Tasse Bleue et le Bol"));
products.data.insert(Map.of(
    "name_en", "A Red Plate with the Saucer",
    "name_fr", "Une Assiette Rouge avec la Soucoupe"));

var products = client.Collections.Use("StopwordsDemo");

await products.Data.Insert(
    new
    {
        name_en = "The Blue Cup and the Bowl",
        name_fr = "La Tasse Bleue et le Bol",
    }
);
await products.Data.Insert(
    new
    {
        name_en = "A Red Plate with the Saucer",
        name_fr = "Une Assiette Rouge avec la Soucoupe",
    }
);

from weaviate.classes.query import MetadataQuery

# Search the French property — "la" and "et" are French stopwords
response = products.query.bm25(
    query="la tasse bleue et le bol",
    query_properties=["name_fr"],
    return_metadata=MetadataQuery(score=True),
)

print("French property search:")
for o in response.objects:
    print(f"  {o.properties['name_fr']} (score: {o.metadata.score})")

# Same words on the English property — "la", "et", "le" are NOT English stopwords
response = products.query.bm25(
    query="la tasse bleue et le bol",
    query_properties=["name_en"],
    return_metadata=MetadataQuery(score=True),
)

print("\nEnglish property search:")
for o in response.objects:
    print(f"  {o.properties['name_en']} (score: {o.metadata.score})")

// Search the French property — "la" and "et" are French stopwords
let response = await products.query.bm25('la tasse bleue et le bol', {
  queryProperties: ['name_fr'],
  returnMetadata: ['score'],
});

console.log('French property search:');
for (const o of response.objects) {
  console.log(`  ${o.properties.name_fr} (score: ${o.metadata?.score})`);
}

// Same words on the English property — "la", "et", "le" are NOT English stopwords
response = await products.query.bm25('la tasse bleue et le bol', {
  queryProperties: ['name_en'],
  returnMetadata: ['score'],
});

console.log('\nEnglish property search:');
for (const o of response.objects) {
  console.log(`  ${o.properties.name_en} (score: ${o.metadata?.score})`);
}

var responseFr = products.query.bm25(
    "la tasse bleue et le bol",
    q -> q.queryProperties("name_fr").returnMetadata(Metadata.SCORE));

System.out.println("French property search:");
for (var o : responseFr.objects()) {
  System.out.println(
      "  " + o.properties().get("name_fr")
          + " (score: " + o.queryMetadata().score() + ")");
}

var responseEn = products.query.bm25(
    "la tasse bleue et le bol",
    q -> q.queryProperties("name_en").returnMetadata(Metadata.SCORE));

System.out.println("\nEnglish property search:");
for (var o : responseEn.objects()) {
  System.out.println(
      "  " + o.properties().get("name_en")
          + " (score: " + o.queryMetadata().score() + ")");
}

var responseFr = await products.Query.BM25(
    query: "la tasse bleue et le bol",
    searchFields: ["name_fr"],
    returnMetadata: MetadataOptions.Score
);

Console.WriteLine("French property search:");
foreach (var o in responseFr.Objects)
{
    Console.WriteLine(
        $"  {o.Properties["name_fr"]} (score: {o.Metadata.Score})"
    );
}

var responseEn = await products.Query.BM25(
    query: "la tasse bleue et le bol",
    searchFields: ["name_en"],
    returnMetadata: MetadataOptions.Score
);

Console.WriteLine("\nEnglish property search:");
foreach (var o in responseEn.Objects)
{
    Console.WriteLine(
        $"  {o.Properties["name_en"]} (score: {o.Metadata.Score})"
    );
}

import weaviate
from weaviate.classes.config import Tokenization, Configure

client = weaviate.connect_to_local()

# Ad-hoc tokenization with custom config
result = client.tokenization.text(
    text="The organic café crème blend",
    tokenization=Tokenization.WORD,
    analyzer_config=Configure.text_analyzer(
        ascii_fold=True,
        stopword_preset="en",
    ),
)

print(f"indexed: {result.indexed}")
print(f"query:   {result.query}")

import weaviate from 'weaviate-client';

const client = await weaviate.connectToLocal();

// Ad-hoc tokenization with custom config
let result = await client.tokenize.text('The organic café crème blend', 'word', {
  analyzerConfig: {
    asciiFold: true,
    stopwordPreset: 'en',
  },
});

console.log(`indexed: ${JSON.stringify(result.indexed)}`);
console.log(`query:   ${JSON.stringify(result.query)}`);

import io.weaviate.client6.v1.api.tokenize.TokenizeResponse;

// Ad-hoc tokenization with custom config
var result = await client.Tokenize.Text(
    text: "The organic café crème blend",
    tokenization: PropertyTokenization.Word,
    analyzerConfig: new TextAnalyzerConfig
    {
        AsciiFold = new AsciiFoldConfig(),
        StopwordPreset = "en",
    }
);

Console.WriteLine($"indexed: [{string.Join(", ", result.Indexed)}]");
Console.WriteLine($"query:   [{string.Join(", ", result.Query)}]");

# Tokenize using an existing property's configuration
result = client.tokenization.for_property(
    collection="TokenizeDemo",
    property_name="name_fr",
    text="La Tasse Bleue et le Bol",
)

print(f"indexed: {result.indexed}")
print(f"query:   {result.query}")

// Tokenize using an existing property's configuration
const propResult = await client.tokenize.forProperty('TokenizeDemo', 'name_fr', 'La Tasse Bleue et le Bol');

console.log(`indexed: ${JSON.stringify(propResult.indexed)}`);
console.log(`query:   ${JSON.stringify(propResult.query)}`);

// Tokenize using an existing property's configuration
TokenizeResponse result = client.tokenize.forProperty(
    "La Tasse Bleue et le Bol", COLLECTION, "name_fr");

System.out.println("indexed: " + result.indexed());
System.out.println("query:   " + result.query());

// Tokenize using an existing property's configuration
var collection = client.Collections.Use(COLLECTION);
var result = await collection.Tokenize.Property(
    propertyName: "name_fr",
    text: "La Tasse Bleue et le Bol"
);

Console.WriteLine($"indexed: [{string.Join(", ", result.Indexed)}]");
Console.WriteLine($"query:   [{string.Join(", ", result.Query)}]");