From 0b6be8f2763eef99d8596140da2c1c48b259a8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Sat, 30 Nov 2024 19:28:51 +0100 Subject: [PATCH 1/5] docs(website): Document CRUD operations --- docs/website/docs/features/crud.md | 177 +++++++++++++++++++++++++++++ docs/website/mkdocs.yml | 3 +- 2 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 docs/website/docs/features/crud.md diff --git a/docs/website/docs/features/crud.md b/docs/website/docs/features/crud.md new file mode 100644 index 00000000..18e8bb6e --- /dev/null +++ b/docs/website/docs/features/crud.md @@ -0,0 +1,177 @@ +# CRUD operations + +CRUD operations (Create, Read, Update and Delete) are the most basic operations a database supports. + +!!! note "" + Before using any of the code in this page, you will need to connect to a database. To learn how to do so, visit the [Getting Started tutorial](../tutorials/index.md). + +MongoDB separates its data into: + +- **Databases** are similar to packages in programming languages. Each database is a namespace providing isolation, so you can deploy multiple projects onto a single MongoDB installation. +- **Collections** are the equivalent of tables in SQL. Each collection can contain an unlimited amount of data. There are various types of collections for our various needs. +- **Documents** are BSON (binary-JSON) objects. MongoDB doesn't verify a schema by default, so they could be completely homogeneous. Since we aim to use Kotlin as a source of truth, however, in practice our Kotlin code serves as the schema. Since documents are JSON, they can contain nested documents and nested arrays, but they cannot exceed 16MB each. + +In MongoDB, CRUD operations target a single collection. +All writes are atomic on the level of a single document. + +## Basic modeling + +With KtMongo, we represent the schema of a collection by declaring a Kotlin class. Depending on the serialization library you are using, this may be slightly different (for example, if you're using KotlinX.Serialization, you'll need to add an `@Serializable` annotation to the class). + +```kotlin +class User( + val name: String, + val age: Int = 0 +) +``` + +MongoDB requires the presence of a unique field named `_id`. If we don't declare it, MongoDB will automatically create it. We can declare the ID to be of any type (including nested documents), but MongoDB is optimized for the special type `ObjectId`: + +```kotlin +class User( + val _id: ObjectId, + val name: String, + val age: Int = 0 +) +``` + +Since we do not use a schema, our Kotlin class represents the source of truth for what the collection can hold. Some serialization libraries (including KotlinX.Serialization) allow using default values in case the field doesn't exist, which allows us to create new fields without breaking the existing data. Similarly, if we remove a field, the existing data is simply ignored. Together, this means migration scripts are rarely needed with MongoDB, unlike with SQL. + +Many people declare these classes as `data class`, the advantage being an improved `toString` representation, but it isn't mandatory. + +In the rest of this article, we assume you have [obtained a collection](../tutorials/index.md) and named it `users`. + +## Create + +Creating a new document is done directly with an instance of the class and the method `insertOne`: + +```kotlin +users.insertOne(User(ObjectId(), "Bob")) +``` + +If the collection didn't yet exist, any write operation creates it. + +If we want to insert multiple documents at the same time, we can use `insertMany`: + +```kotlin +users.insertMany( + User(ObjectId(), "Bob"), + User(ObjectId(), "Marcel"), + User(ObjectId(), "Jeanne") +) +``` + +## Read + +Read operations retrieve documents from a collection. +For example, we can `count` how many documents exist in a collection: + +```kotlin +users.count() +``` + +Or, we can get all the documents using the `find` method: + +```kotlin +users.find().toList() +``` + +However, lists are in-memory data structures, and it may not be appropriate to query an entire collection into memory. Instead, we can stream the results using `forEach`: + +```kotlin +users.find().forEach { println("Found a document: $it") } +``` + +Or, if we want to further process the documents, we can use asynchronous steaming functionalities: + +=== "Coroutines driver" + + ```kotlin + users.find().asFlow() + ``` + +=== "Synchronous driver" + + ```kotlin + users.find().asStream() + ``` + +Of course, we usually want to let the database perform filters, as it benefits from indexes. Filters are declared in a trailing lambda, usually as infix functions. Filters apply to a specific field, which is referred to using the name of the class, followed by `::`, followed by the name of the field: + +```kotlin +users.find { + User::name gte "C" + User::name lt "G" +}.toList() +``` + +This query will return all users with a name that is alphabetically between "C" and "G". +This syntax is typesafe: invalid requests (for example comparing against another type) will not compile. + +[//]: # (TODO: add a link to the 'collation' option, whenever it is implemented) + +If you are only interested in a single document, use `findOne`, which returns a nullable value instead of a list: + +```kotlin +users.findOne { + User::name eq "Bob" +} +``` + +Learn more: + +- [Referring to fields](fields.md) + +## Update + +Update operations modify existing documents in a collection. + +Similarly to search criteria, we can use infix operators to update some fields. To update all documents, use `updateMany`: + +```kotlin +users.updateMany { + User::age inc 1 +} +``` + +If you want to only edit some documents (not the entire collection), use the optional `filter` parameter, which accepts the same syntax as `find()` and `findOne()`: + +```kotlin +users.updateMany( + filter = { + User::name eq "Bob" + } +) { + User::age inc 1 +} +``` + +If you only want to update a single document, use `updateOne` instead, which has the same syntax. + +Finally, if you want to ensure that a specific document exists, and want to create it if it doesn't, use `upsertOne`. + +## Delete + +Delete operations remove documents from a collection. Delete operations accept a filter, just like `findOne` and `findMany`. + +To delete one document, use `deleteOne`: + +```kotlin +users.deleteOne { + User::name eq "Bob" +} +``` + +To delete multiple documents, use `deleteMany`: + +```kotlin +users.deleteMany { + User::age lt 18 +} +``` + +Additionally, to delete the entire collection, use `drop`: + +```kotlin +users.drop() +``` diff --git a/docs/website/mkdocs.yml b/docs/website/mkdocs.yml index 0bda2e46..4dc7f28b 100644 --- a/docs/website/mkdocs.yml +++ b/docs/website/mkdocs.yml @@ -96,7 +96,8 @@ nav: - tutorials/from-kmongo/nested-fields.md - tutorials/from-kmongo/update.md -# - Features: [] + - Features: + - features/crud.md - Reference: - reference.md -- 2.51.2 From 042d0279313f83e517d575e56ee2dbc655b5cee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Sat, 30 Nov 2024 19:58:40 +0100 Subject: [PATCH 2/5] docs(website): Document bulk writes --- docs/website/docs/features/bulk-writes.md | 59 +++++++++++++++++++++++ docs/website/mkdocs.yml | 1 + 2 files changed, 60 insertions(+) create mode 100644 docs/website/docs/features/bulk-writes.md diff --git a/docs/website/docs/features/bulk-writes.md b/docs/website/docs/features/bulk-writes.md new file mode 100644 index 00000000..7e1990c0 --- /dev/null +++ b/docs/website/docs/features/bulk-writes.md @@ -0,0 +1,59 @@ +# Bulk writes + +Changing one document at a time can be quite expensive because of the high network activity and high latencies. Instead, when we know we want to edit multiple documents, we prefer to do so in a single request. + +For example, this is **bad code** that should be avoided: + +```kotlin title="Bad example!" +val usersToCreate = listOf( + User("Bob"), + User("Marcel"), + /* … */ +) + +for (user in usersToCreate) { + users.insertOne(user) +} +``` + +This code is bad because each insert will send data to the database and wait for its response. Between each insert, it waits for the previous one to finish and for an entire network roundtrip. + +Instead, we can insert all users at once with `insertMany`: + +```kotlin +val usersToCreate = listOf( + User("Bob"), + User("Marcel"), + /* … */ +) + +users.insertMany(usersToCreate) +``` + +Here, a single request is sent to the database, which can perform all inserts much quicker. + +Similarly, other write operations have a variant that allows performing the same write on multiple documents: `updateMany` and `deleteMany`. + +Sometimes, however, we want to perform very different writes, but we could still benefit from sending them all in a single request. In those situations, we can use `bulkWrite`: + +```kotlin +users.bulkWrite { + insertMany(usersToCreate) + + updateOne( + filter = { User::name eq "Bob" } + ) { + User::age set 65 + } + + updateMany { + User::age inc 1 + } + + deleteOne { + User::name eq "Janine" + } +} +``` + +Note that this _isn't_ a transaction. The operations are performed in the same way they would be if calling their respective methods, the only difference is they are all sent together to the database in a single request to decrease network traffic and latency. diff --git a/docs/website/mkdocs.yml b/docs/website/mkdocs.yml index 4dc7f28b..7201da15 100644 --- a/docs/website/mkdocs.yml +++ b/docs/website/mkdocs.yml @@ -98,6 +98,7 @@ nav: - Features: - features/crud.md + - features/bulk-writes.md - Reference: - reference.md -- 2.51.2 From da28321d6061664244d439f6f3abdd25825b807f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Sat, 30 Nov 2024 20:41:43 +0100 Subject: [PATCH 3/5] docs(website): Document nested field accesses --- docs/website/docs/features/fields.md | 142 +++++++++++++++++++++++++++ docs/website/mkdocs.yml | 1 + 2 files changed, 143 insertions(+) create mode 100644 docs/website/docs/features/fields.md diff --git a/docs/website/docs/features/fields.md b/docs/website/docs/features/fields.md new file mode 100644 index 00000000..e25b3045 --- /dev/null +++ b/docs/website/docs/features/fields.md @@ -0,0 +1,142 @@ +# Fields and nested documents + +As we have seen in the [CRUD operations](crud.md) article, KtMongo operators apply to specific fields in a document. This article describes the different ways in which we can refer to these fields. + +Note that field access is type-safe: accessing to a field in an invalid way, or accessing to a field in a context in which it cannot be accessed, will lead to compilation errors. + +## In the root document + +Each collection is declared on a specific document class. That document class is called the root document. + +When we want to refer to fields from the root document, we simply type the root document's class name, followed by `::`, followed by the field's name: + +```kotlin hl_lines="7" +class User( + val name: String, + val age: Int? +) + +users.find { + User::name eq "John" +} +``` + +Note that referring to fields from the wrong class will not compile. + +## In nested documents + +If we have nested documents, we can refer to any field using the `/` operator (similar to file paths): +```kotlin hl_lines="13" +class User( + val _id: ObjectId, + val profile: Profile, + val hashedPassword: String, +) + +class Profile( + val name: String, + val age: Int? +) + +users.find { + User::profile / Profile::name eq "John" +} +``` + +Again, this syntax is fully typed. + +## In arrays + +Documents can also be stored in arrays. In Kotlin, BSON arrays can be represented using any collection type, like `List` or `Set`. + +### Using an index + +The easiest way to access an array item is using its index. For example, if we know that we want to update Bob's second-best friend, we can use: +```kotlin hl_lines="14" +class User( + val name: String, + val friends: List, +) + +class Friend( + val name: String, + val preferredFood: String, +) + +users.updateOne( + filter = { User::name eq "Bob" } +) { + User::friends[1] / Friend::preferredFood set "Lasagna" +} +``` + +### Based on its properties + +We may be interested in searching for a document based on the properties of an item in an array. For example, if we want to find all students who have had a perfect grade, we can use the `any` operator: +```kotlin hl_lines="7" +class Student( + val name: String, + val grades: List, +) + +students.find { + Student::grades.any eq 20 +} +``` +This reads as: "find all students from which any grade is 20". If we want to find all students that also have the worst possible grade, we can write: +```kotlin +students.find { + Student::grades.any eq 20 + Student::grades.any eq 0 +} +``` +This finds all students that have a grade of 20, and another grade of 0. + +If, instead, we want to provide multiple filters on _the same grade_, we can use the `anyValue` operator: +```kotlin +students.find { + Student::grades.anyValue { + gt(18) + lte(19) + } +} +``` + +### Based on the properties of a nested document + +This section is the same as the previous one, but instead of filtering on an item itself, we filter based on the field of the item. + +If we want to find all users who have a pet named Lucy: +```kotlin +class User( + val name: String, + val pets: List, +) + +class Pet( + val name: String, + val age: Int, +) + +users.find { + User::pets.any / Pet::name eq "Lucy" +} +``` + +If we want to find all users who have a pet named Lucy, and also a pet that is 4 years old, but these could be two different pets: +```kotlin +users.find { + User::pets.any / Pet::name eq "Lucy" + User::pets.any / Pet::age eq 4 +} +``` + +If we want to find all users who have a pet named Lucy that is 4 years old (must be the same pet): +```kotlin +users.find { + User::pets.any { + Pet::name eq "Lucy" + Pet::age eq 4 + } +} +``` diff --git a/docs/website/mkdocs.yml b/docs/website/mkdocs.yml index 7201da15..e6b4e287 100644 --- a/docs/website/mkdocs.yml +++ b/docs/website/mkdocs.yml @@ -99,6 +99,7 @@ nav: - Features: - features/crud.md - features/bulk-writes.md + - features/fields.md - Reference: - reference.md -- 2.51.2 From d236b01ea263c8d78913ca6af481bae7f74d4e28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Sat, 30 Nov 2024 21:57:10 +0100 Subject: [PATCH 4/5] feat(dsl): Implement the .$. (positional) operator in updates --- docs/website/docs/features/fields.md | 29 ++++++ .../commonMain/kotlin/expr/UpdateOperators.kt | 90 ++++++++++++++++++- test/src/commonTest/kotlin/ArraysTest.kt | 41 +++++++++ 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/docs/website/docs/features/fields.md b/docs/website/docs/features/fields.md index e25b3045..134c0c25 100644 --- a/docs/website/docs/features/fields.md +++ b/docs/website/docs/features/fields.md @@ -102,6 +102,20 @@ students.find { } ``` +Now that we are able to select a specific grade, we can update it using the `selected` operator. For example, if we wanted to increase that grade: +```kotlin +students.updateOne( + filter = { + Student::grades.anyValue { + gt(18) + lte(19) + } + } +) { + Student::grades.selected inc 1 +} +``` + ### Based on the properties of a nested document This section is the same as the previous one, but instead of filtering on an item itself, we filter based on the field of the item. @@ -140,3 +154,18 @@ users.find { } } ``` + +Now that we are able to select a specific pet, we can use the `selected` operator to update it: +```kotlin +users.updateOne( + filter = { + User::pets.any { + Pet::name eq "Lucy" + Pet::age eq 4 + } + } +) { + User::pets.selected / Pet::age inc 1 +} +``` +Only the pet we referred to will be updated. diff --git a/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt b/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt index 5567345d..19c078e6 100644 --- a/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt +++ b/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt @@ -20,8 +20,7 @@ import opensavvy.ktmongo.dsl.DangerousMongoApi import opensavvy.ktmongo.dsl.KtMongoDsl import opensavvy.ktmongo.dsl.LowLevelApi import opensavvy.ktmongo.dsl.expr.common.CompoundExpression -import opensavvy.ktmongo.dsl.path.Field -import opensavvy.ktmongo.dsl.path.FieldDsl +import opensavvy.ktmongo.dsl.path.* import kotlin.reflect.KProperty1 /** @@ -56,6 +55,7 @@ import kotlin.reflect.KProperty1 * - [`$unset`][unset] * * On arrays: + * - [`$`][selected] * - [`$[]`][FieldDsl.get] * * If you can't find the operator you're searching for, visit the [tracking issue](https://gitlab.com/opensavvy/ktmongo/-/issues/5). @@ -374,6 +374,92 @@ interface UpdateOperators : CompoundExpression, FieldDsl { } // endregion + // region Positional operator: $ + + /** + * The positional operator: update an array item selected in the filter. + * + * When we use [any][FilterOperators.any] or [anyValue][FilterOperators.anyValue] + * in a filter to select an item, we can use this operator to update whichever item was selected. + * + * Do not use this operator in an `upsert`. + * + * ### Example + * + * ```kotlin + * class User( + * val name: String, + * val pets: List, + * ) + * + * class Pet( + * val name: String, + * val age: Int, + * ) + * + * users.updateMany( + * filter = { + * User::pets.any / Pet::name eq "Bobby" + * }, + * update = { + * User::pets.selected / Pet::age inc 1 + * } + * ) + * ``` + * + * This example finds all users who have a pet named "Bobby", and increases its age by 1. + * Note that if the users have other pets, they are not impacted. + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/update/positional/) + */ + @OptIn(LowLevelApi::class) + val Field>.selected: Field + get() = FieldImpl(path / PathSegment.Positional) + + /** + * The positional operator: update an array item selected in the filter. + * + * When we use [any][FilterOperators.any] or [anyValue][FilterOperators.anyValue] + * in a filter to select an item, we can use this operator to update whichever item was selected. + * + * Do not use this operator in an `upsert`. + * + * ### Example + * + * ```kotlin + * class User( + * val name: String, + * val pets: List, + * ) + * + * class Pet( + * val name: String, + * val age: Int, + * ) + * + * users.updateMany( + * filter = { + * User::pets.any / Pet::name eq "Bobby" + * }, + * update = { + * User::pets.selected / Pet::age inc 1 + * } + * ) + * ``` + * + * This example finds all users who have a pet named "Bobby", and increases its age by 1. + * Note that if the users have other pets, they are not impacted. + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/update/positional/) + */ + val KProperty1>.selected: Field + get() = this.field.selected + + // endregion } diff --git a/test/src/commonTest/kotlin/ArraysTest.kt b/test/src/commonTest/kotlin/ArraysTest.kt index 6b3b9603..83dd2c90 100644 --- a/test/src/commonTest/kotlin/ArraysTest.kt +++ b/test/src/commonTest/kotlin/ArraysTest.kt @@ -56,4 +56,45 @@ class ArraysTest : PreparedSpec({ } } + test("Position operator: $") { + @Serializable + data class Pet( + val name: String, + val age: Int, + ) + + @Serializable + data class Profile( + val name: String, + val pets: List, + ) + + val profiles = testCollection("arrays-positional") + .immediate("profiles") + + val initial = listOf( + Profile("Bob", listOf(Pet("Bobby", 1), Pet("Cacahuète", 10))), + Profile("Julia", listOf(Pet("Chouquette", 7))) + ) + + profiles.insertMany(initial) + + // Cacahuète got one year older + val expected = listOf( + Profile("Bob", listOf(Pet("Bobby", 1), Pet("Cacahuète", 11))), + Profile("Julia", listOf(Pet("Chouquette", 7))) + ) + + profiles.updateMany( + filter = { + Profile::pets.any / Pet::name eq "Cacahuète" + }, + update = { + Profile::pets.selected / Pet::age inc 1 + } + ) + + check(expected == profiles.find().toList()) + } + }) -- 2.51.2 From 77814611164c6d580d942fb89a191a9152c9e361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Sun, 1 Dec 2024 10:50:32 +0100 Subject: [PATCH 5/5] feat(dsl): Implement the .$[]. (all positional) operator in updates --- docs/website/docs/features/fields.md | 19 ++++++ .../commonMain/kotlin/expr/UpdateOperators.kt | 66 ++++++++++++++++++- test/src/commonTest/kotlin/ArraysTest.kt | 36 ++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docs/website/docs/features/fields.md b/docs/website/docs/features/fields.md index 134c0c25..19d5da11 100644 --- a/docs/website/docs/features/fields.md +++ b/docs/website/docs/features/fields.md @@ -169,3 +169,22 @@ users.updateOne( } ``` Only the pet we referred to will be updated. + +### Update all elements + +If we want to update all elements in an array, we can use the `all` operator: +```kotlin +class User( + val name: String, + val pets: List, +) + +class Pet( + val name: String, + val age: Int, +) + +users.updateMany { + User::pets.all / Pet::age set 1 +} +``` diff --git a/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt b/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt index 19c078e6..c58629a5 100644 --- a/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt +++ b/dsl/src/commonMain/kotlin/expr/UpdateOperators.kt @@ -56,7 +56,7 @@ import kotlin.reflect.KProperty1 * * On arrays: * - [`$`][selected] - * - [`$[]`][FieldDsl.get] + * - [`$[]`][all] * * If you can't find the operator you're searching for, visit the [tracking issue](https://gitlab.com/opensavvy/ktmongo/-/issues/5). * @@ -460,6 +460,70 @@ interface UpdateOperators : CompoundExpression, FieldDsl { get() = this.field.selected // endregion + // region All positional operator: $[] + + /** + * The all positional operator: selects all elements of an array. + * + * This operator is used to declare an update that applies to all items of an array. + * + * ### Example + * + * ```kotlin + * class User( + * val name: String, + * val pets: List, + * ) + * + * class Pet( + * val name: String, + * val age: Int, + * ) + * + * users.updateMany { + * User::pets.all / Pet::age inc 1 + * } + * ``` + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/update/positional-all/) + */ + @OptIn(LowLevelApi::class) + val Field>.all: Field + get() = FieldImpl(path / PathSegment.AllPositional) + + /** + * The all positional operator: selects all elements of an array. + * + * This operator is used to declare an update that applies to all items of an array. + * + * ### Example + * + * ```kotlin + * class User( + * val name: String, + * val pets: List, + * ) + * + * class Pet( + * val name: String, + * val age: Int, + * ) + * + * users.updateMany { + * User::pets.all / Pet::age inc 1 + * } + * ``` + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/update/positional-all/) + */ + val KProperty1>.all: Field + get() = this.field.all + + // endregion } diff --git a/test/src/commonTest/kotlin/ArraysTest.kt b/test/src/commonTest/kotlin/ArraysTest.kt index 83dd2c90..1306678e 100644 --- a/test/src/commonTest/kotlin/ArraysTest.kt +++ b/test/src/commonTest/kotlin/ArraysTest.kt @@ -97,4 +97,40 @@ class ArraysTest : PreparedSpec({ check(expected == profiles.find().toList()) } + test("All position operator: $[]") { + @Serializable + data class Pet( + val name: String, + val age: Int, + ) + + @Serializable + data class Profile( + val name: String, + val pets: List, + ) + + val profiles = testCollection("arrays-positional") + .immediate("profiles") + + val initial = listOf( + Profile("Bob", listOf(Pet("Bobby", 1), Pet("Cacahuète", 10))), + Profile("Julia", listOf(Pet("Chouquette", 7))) + ) + + profiles.insertMany(initial) + + // All pets get one year older + val expected = listOf( + Profile("Bob", listOf(Pet("Bobby", 2), Pet("Cacahuète", 11))), + Profile("Julia", listOf(Pet("Chouquette", 8))) + ) + + profiles.updateMany { + Profile::pets.all / Pet::age inc 1 + } + + check(expected == profiles.find().toList()) + } + }) -- 2.51.2