diff --git a/ai-assistance.mdx b/ai-assistance.mdx index 746ec14..fdae84d 100644 --- a/ai-assistance.mdx +++ b/ai-assistance.mdx @@ -24,7 +24,7 @@ After you upload the [Documentation Context](https://docs.tilebox.com/llms-full. Here's an example prompt to get you started. -```txt Example prompt +```plaintext Example prompt Generate a script that - Creates a cluster diff --git a/api-reference/tilebox.workflows/Client.mdx b/api-reference/tilebox.workflows/Client.mdx index bd45ea0..873c7e0 100644 --- a/api-reference/tilebox.workflows/Client.mdx +++ b/api-reference/tilebox.workflows/Client.mdx @@ -36,10 +36,10 @@ def Client.clusters() -> ClusterClient A client for managing clusters. ```python -def Client.recurrent_tasks() -> RecurrentTaskClient +def Client.automations() -> AutomationClient ``` -A client for scheduling recurrent tasks. +A client for scheduling automations. ## Task runners @@ -65,7 +65,7 @@ client = Client( # access sub clients job_client = client.jobs() cluster_client = client.clusters() -recurrent_task_client = client.recurrent_tasks() +automation_client = client.automations() # or instantiate a task runner runner = client.runner("dev-cluster", tasks=[...]) diff --git a/assets/console/api-keys-dark.png b/assets/console/api-keys-dark.png index d77efd5..ef3cff9 100644 Binary files a/assets/console/api-keys-dark.png and b/assets/console/api-keys-dark.png differ diff --git a/assets/console/api-keys-light.png b/assets/console/api-keys-light.png index 05f5f57..f04ccc6 100644 Binary files a/assets/console/api-keys-light.png and b/assets/console/api-keys-light.png differ diff --git a/assets/console/automation-dark.png b/assets/console/automation-dark.png new file mode 100644 index 0000000..bca7ef2 Binary files /dev/null and b/assets/console/automation-dark.png differ diff --git a/assets/console/automation-edit-dark.png b/assets/console/automation-edit-dark.png new file mode 100644 index 0000000..3df9a73 Binary files /dev/null and b/assets/console/automation-edit-dark.png differ diff --git a/assets/console/automation-edit-light.png b/assets/console/automation-edit-light.png new file mode 100644 index 0000000..acd2614 Binary files /dev/null and b/assets/console/automation-edit-light.png differ diff --git a/assets/console/automation-light.png b/assets/console/automation-light.png new file mode 100644 index 0000000..bf12580 Binary files /dev/null and b/assets/console/automation-light.png differ diff --git a/assets/console/automations-dark.png b/assets/console/automations-dark.png new file mode 100644 index 0000000..87ef367 Binary files /dev/null and b/assets/console/automations-dark.png differ diff --git a/assets/console/automations-light.png b/assets/console/automations-light.png new file mode 100644 index 0000000..0488992 Binary files /dev/null and b/assets/console/automations-light.png differ diff --git a/assets/console/cluster-dark.png b/assets/console/cluster-dark.png index 16b6a9a..b437301 100644 Binary files a/assets/console/cluster-dark.png and b/assets/console/cluster-dark.png differ diff --git a/assets/console/cluster-light.png b/assets/console/cluster-light.png index d15f8d0..091ac61 100644 Binary files a/assets/console/cluster-light.png and b/assets/console/cluster-light.png differ diff --git a/assets/console/datapoints-dark.png b/assets/console/datapoints-dark.png new file mode 100644 index 0000000..1b44a65 Binary files /dev/null and b/assets/console/datapoints-dark.png differ diff --git a/assets/console/datapoints-light.png b/assets/console/datapoints-light.png new file mode 100644 index 0000000..be35e7e Binary files /dev/null and b/assets/console/datapoints-light.png differ diff --git a/assets/console/datasets-dark.png b/assets/console/datasets-dark.png index e90c66d..38d6e51 100644 Binary files a/assets/console/datasets-dark.png and b/assets/console/datasets-dark.png differ diff --git a/assets/console/datasets-light.png b/assets/console/datasets-light.png index 57672e0..5a348e9 100644 Binary files a/assets/console/datasets-light.png and b/assets/console/datasets-light.png differ diff --git a/assets/console/recurrent-task-dark.png b/assets/console/recurrent-task-dark.png deleted file mode 100644 index e748542..0000000 Binary files a/assets/console/recurrent-task-dark.png and /dev/null differ diff --git a/assets/console/recurrent-task-light.png b/assets/console/recurrent-task-light.png deleted file mode 100644 index 10bee35..0000000 Binary files a/assets/console/recurrent-task-light.png and /dev/null differ diff --git a/assets/console/storage-event-dark.png b/assets/console/storage-event-dark.png new file mode 100644 index 0000000..5cbbcb6 Binary files /dev/null and b/assets/console/storage-event-dark.png differ diff --git a/assets/console/storage-event-light.png b/assets/console/storage-event-light.png new file mode 100644 index 0000000..5989d52 Binary files /dev/null and b/assets/console/storage-event-light.png differ diff --git a/assets/console/usage-dark.png b/assets/console/usage-dark.png index 04413ce..884a3a1 100644 Binary files a/assets/console/usage-dark.png and b/assets/console/usage-dark.png differ diff --git a/assets/console/usage-light.png b/assets/console/usage-light.png index 0e91b33..023a8b6 100644 Binary files a/assets/console/usage-light.png and b/assets/console/usage-light.png differ diff --git a/assets/workflows/recurrent-tasks/cron-dark.png b/assets/workflows/recurrent-tasks/cron-dark.png deleted file mode 100644 index 8fce462..0000000 Binary files a/assets/workflows/recurrent-tasks/cron-dark.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/cron-light.png b/assets/workflows/recurrent-tasks/cron-light.png deleted file mode 100644 index 51e9153..0000000 Binary files a/assets/workflows/recurrent-tasks/cron-light.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/recurrent-task-edit-dark.png b/assets/workflows/recurrent-tasks/recurrent-task-edit-dark.png deleted file mode 100644 index 3dad053..0000000 Binary files a/assets/workflows/recurrent-tasks/recurrent-task-edit-dark.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/recurrent-task-edit-light.png b/assets/workflows/recurrent-tasks/recurrent-task-edit-light.png deleted file mode 100644 index b4150cc..0000000 Binary files a/assets/workflows/recurrent-tasks/recurrent-task-edit-light.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/recurrent-tasks-dark.png b/assets/workflows/recurrent-tasks/recurrent-tasks-dark.png deleted file mode 100644 index 8c46343..0000000 Binary files a/assets/workflows/recurrent-tasks/recurrent-tasks-dark.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/recurrent-tasks-light.png b/assets/workflows/recurrent-tasks/recurrent-tasks-light.png deleted file mode 100644 index 7c959f3..0000000 Binary files a/assets/workflows/recurrent-tasks/recurrent-tasks-light.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/storage-event-dark.png b/assets/workflows/recurrent-tasks/storage-event-dark.png deleted file mode 100644 index 76a6574..0000000 Binary files a/assets/workflows/recurrent-tasks/storage-event-dark.png and /dev/null differ diff --git a/assets/workflows/recurrent-tasks/storage-event-light.png b/assets/workflows/recurrent-tasks/storage-event-light.png deleted file mode 100644 index 6b1053e..0000000 Binary files a/assets/workflows/recurrent-tasks/storage-event-light.png and /dev/null differ diff --git a/console.mdx b/console.mdx index 0a1b8e9..da550b5 100644 --- a/console.mdx +++ b/console.mdx @@ -45,11 +45,11 @@ data = sentinel2_msi.collection("S2A_S2MSI1C").load( ## Workflows -The workflows section of the console allows you to view jobs, create clusters and create recurring and near real-time tasks. +The workflows section of the console allows you to view jobs, create clusters and create near real-time automations. - Tilebox Console - Tilebox Console + Tilebox Console + Tilebox Console ## Account diff --git a/mint.json b/mint.json index 212a0ee..09c38ab 100644 --- a/mint.json +++ b/mint.json @@ -136,7 +136,7 @@ "group": "Near-Real Time", "icon": "bolt", "pages": [ - "workflows/near-real-time/recurrent-tasks", + "workflows/near-real-time/automations", "workflows/near-real-time/cron", "workflows/near-real-time/storage-events" ] diff --git a/vale/styles/config/vocabularies/docs/accept.txt b/vale/styles/config/vocabularies/docs/accept.txt index 7ec28c5..79654fe 100644 --- a/vale/styles/config/vocabularies/docs/accept.txt +++ b/vale/styles/config/vocabularies/docs/accept.txt @@ -46,3 +46,4 @@ coroutines (?i)Imager (?i)pushbroom rollout +(?i)automations diff --git a/workflows/introduction.mdx b/workflows/introduction.mdx index f5d6c3e..0033a90 100644 --- a/workflows/introduction.mdx +++ b/workflows/introduction.mdx @@ -24,7 +24,7 @@ This section provides guides showcasing how to use the Tilebox workflow orchestr Learn to configure shared data access for all tasks of a job using caches. - + Trigger jobs based on events or schedules, such as new data availability or CRON schedules. diff --git a/workflows/near-real-time/automations.mdx b/workflows/near-real-time/automations.mdx new file mode 100644 index 0000000..7a70484 --- /dev/null +++ b/workflows/near-real-time/automations.mdx @@ -0,0 +1,92 @@ +--- +title: Automations +description: Process data in near-real-time by triggering jobs based on external events +icon: repeat +--- + +## Introduction + +Tilebox Workflows can execute jobs in two ways: a one-time execution triggered by a user, typically a batch processing, and near-real-time execution based on specific external events. +By defining trigger conditions, you can automatically submit jobs based on external events. + +Tilebox Workflows currently supports the following trigger conditions: + + + + Trigger jobs based on a Cron schedule. + + + Trigger jobs after objects are created or modified in a storage location such as a cloud bucket. + + + + + Dataset Event Triggers, which will trigger jobs when new data points are ingested into a Tilebox dataset, are on the roadmap. Stay tuned for updates. + + +## Automations + +To create a trigger, define a special task that serves as a prototype. In response to a trigger condition met, this task will be submitted as a new job. Such tasks are referred to as automations. + +Each automation has a [task identifier](/workflows/concepts/tasks#task-identifiers), a [version](/workflows/concepts/tasks#semantic-versioning), and [input parameters](/workflows/concepts/tasks#input-parameters), just like regular tasks. +Automations also automatically provide a special `trigger` attribute that contains information about the event that initiated the task's execution. + +## Automation Client + +The Tilebox Workflows client includes a sub-client for managing automations. You can create this sub-client by calling the `automations` method on the main client instance. + +### Listing Registered Automations + +To list all registered automations, use the `all` method on the automation client. + +```python Python +from tilebox.workflows import Client + +client = Client() +automations = client.automations() + +automations = automations.all() +print(automations) +``` + +```plaintext Output +[ + AutomationPrototype( + name='Run MyCronTask every hour at 15 minutes past the hour', + prototype=TaskSubmission( + cluster_slug='dev-cluster', + identifier=TaskIdentifier(name='MyCronTask', version='v0.0'), + input=b'{"message": "Hello"}, + dependencies=[], + display='MyCronTask', + max_retries=0), + storage_event_triggers=[], + cron_triggers=[CronTrigger(schedule='15 * * * *')], + ) +] +``` + +### Registering Automations + +To register an automation, use the `create_*_automation` methods specific to each trigger type provided by the automation client. Refer to the documentation for each trigger type for more details. + + + + + + +## Overview in the Tilebox Console + +You can also use the Tilebox Console to manage automations. Visit [the automations section](https://console.tilebox.com/workflows/automations) to check it out. + + + Tilebox Workflows automations in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console + + +You can also register new automations or edit and delete existing ones directly from the console. + + + Tilebox Workflows automations in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console + diff --git a/workflows/near-real-time/cron.mdx b/workflows/near-real-time/cron.mdx index d6f8e8c..493437c 100644 --- a/workflows/near-real-time/cron.mdx +++ b/workflows/near-real-time/cron.mdx @@ -6,11 +6,12 @@ icon: clock ## Creating Cron tasks -Cron tasks run repeatedly on a specified [cron](https://en.wikipedia.org/wiki/Cron) schedule. To create a Cron task, use `tilebox.workflows.recurrent_tasks.CronTask` as your tasks base class instead of the regular `tilebox.workflows.Task`. +Cron tasks run repeatedly on a specified [cron](https://en.wikipedia.org/wiki/Cron) schedule. +To create a Cron task, use `tilebox.workflows.automations.CronTask` as your tasks base class instead of the regular `tilebox.workflows.Task`. ```python Python from tilebox.workflows import ExecutionContext -from tilebox.workflows.recurrent_tasks import CronTask +from tilebox.workflows.automations import CronTask class MyCronTask(CronTask): message: str @@ -25,15 +26,16 @@ class MyCronTask(CronTask): ## Registering a Cron trigger -After implementing a Cron task, register it to be triggered according to a Cron schedule. When the Cron expression matches, a new job is submitted consisting of a single task instance derived from the Cron task prototype. +After implementing a Cron task, register it to be triggered according to a Cron schedule. +When the Cron expression matches, a new job is submitted consisting of a single task instance derived from the Cron task prototype. ```python Python from tilebox.workflows import Client client = Client() -recurrent_tasks = client.recurrent_tasks() -cron_task = recurrent_tasks.create_recurring_cron_task( - "my-cron-task", # name of the recurring cron task +automations = client.automations() +cron_automation = automations.create_cron_automation( + "my-cron-automation", # name of the cron automation "dev-cluster", # cluster slug to submit jobs to MyCronTask(message="World"), # the task (and its input parameters) to run repeatedly cron_triggers=[ @@ -51,7 +53,7 @@ cron_task = recurrent_tasks.create_recurring_cron_task( ## Starting a Cron Task Runner -With the Cron task registered, a job is submitted whenever the Cron expression matches. But unless a [task runner](/workflows/concepts/task-runners) is available to execute the Cron task the submitted jobs remain in a task queue. +With the Cron automation registered, a job is submitted whenever the Cron expression matches. But unless a [task runner](/workflows/concepts/task-runners) is available to execute the Cron task the submitted jobs remain in a task queue. Once an [eligible task runner](/workflows/concepts/task-runners#task-selection) becomes available, all jobs in the queue are executed. ```python Python @@ -79,30 +81,30 @@ This task was triggered at 2023-09-25 19:12:00 ## Inspecting in the Console -The [Tilebox Console](https://console.tilebox.com/workflows/recurrent-tasks) provides a straightforward way to inspect all registered Cron tasks. +The [Tilebox Console](https://console.tilebox.com/workflows/automations) provides a straightforward way to inspect all registered Cron automations. - Tilebox Workflows recurrent tasks in the Tilebox Console - Tilebox Workflows recurrent tasks in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console -Use the console to view, edit, and delete the registered Cron tasks. +Use the console to view, edit, and delete the registered Cron automations. -## Deleting Cron triggers +## Deleting Cron automations -To delete a registered Cron task, use `recurrent_tasks.delete`. After deletion, no new jobs will be submitted by that Cron trigger. Past jobs already triggered will still remain queued. +To delete a registered Cron automation, use `automations.delete`. After deletion, no new jobs will be submitted by that Cron trigger. Past jobs already triggered will still remain queued. ```python Python from tilebox.workflows import Client client = Client() -recurrent_tasks = client.recurrent_tasks() +automations = client.automations() -# delete the task as returned by create_recurring_cron_task -recurrent_tasks.delete(cron_task) +# delete the automation as returned by create_cron_automation +automations.delete(cron_automation) # or manually by id: -recurrent_tasks.delete("0190bafc-b3b8-88c4-008b-a5db044380d0") +automations.delete("0190bafc-b3b8-88c4-008b-a5db044380d0") ``` ## Submitting Cron jobs manually diff --git a/workflows/near-real-time/recurrent-tasks.mdx b/workflows/near-real-time/recurrent-tasks.mdx deleted file mode 100644 index 8dfbad5..0000000 --- a/workflows/near-real-time/recurrent-tasks.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: Recurrent Tasks -description: Process data in near-real-time by triggering jobs based on external events -icon: repeat ---- - -## Introduction - -Tilebox Workflows can execute jobs in two ways: a one-time execution triggered by a user, typically a batch processing, and near-real-time execution based on specific external events. By defining trigger conditions, you can automatically submit jobs based on external events. - -Tilebox Workflows currently supports the following trigger conditions: - - - - Trigger jobs based on a Cron schedule. - - - Trigger jobs after objects are created or modified in a storage location such as a cloud bucket. - - - - - Dataset Event Triggers, which will trigger jobs when new data points are ingested into a Tilebox dataset, are on the roadmap. Stay tuned for updates. - - -## Recurrent Tasks - -To create a trigger, define a special task that serves as a prototype. In response to a trigger condition met, this task will be submitted as a new job. Such tasks are referred to as recurrent tasks. - -Each recurrent task has a [task identifier](/workflows/concepts/tasks#task-identifiers), a [version](/workflows/concepts/tasks#semantic-versioning), and [input parameters](/workflows/concepts/tasks#input-parameters), just like regular tasks. -Recurrent tasks also automatically provide a special `trigger` attribute that contains information about the event that initiated the task's execution. - -## Recurrent Task Client - -The Tilebox Workflows client includes a sub-client for managing recurrent tasks. You can create this sub-client by calling the `recurrent_tasks` method on the main client instance. - -### Listing Registered Recurrent Tasks - -To list all registered recurrent tasks, use the `all` method on the recurrent task client. - -```python Python -from tilebox.workflows import Client - -client = Client() -recurrent_tasks = client.recurrent_tasks() - -recurrent_tasks = recurrent_tasks.all() -print(recurrent_tasks) -``` - -```plaintext Output -[ - RecurrentTaskPrototype( - name='Run MyCronTask every hour at 15 minutes past the hour', - prototype=TaskSubmission( - cluster_slug='dev-cluster', - identifier=TaskIdentifier(name='MyCronTask', version='v0.0'), - input=b'{"message": "Hello"}, - dependencies=[], - display='MyCronTask', - max_retries=0), - storage_event_triggers=[], - cron_triggers=[CronTrigger(schedule='15 * * * *')], - ) -] -``` - -### Registering Recurrent Tasks - -To register a recurrent task, use the `create_recurring_...` methods specific to each trigger type provided by the recurrent task client. Refer to the documentation for each trigger type for more details. - - - - - - -## Overview in the Tilebox Console - -You can also use the Tilebox Console to manage recurrent tasks. Visit [the recurrent tasks section](https://console.tilebox.com/workflows/recurrent-tasks) to check it out. - - - Tilebox Workflows recurrent tasks in the Tilebox Console - Tilebox Workflows recurrent tasks in the Tilebox Console - - -You can also register new recurrent tasks or edit and delete existing ones directly from the console. - - - Tilebox Workflows recurrent tasks in the Tilebox Console - Tilebox Workflows recurrent tasks in the Tilebox Console - diff --git a/workflows/near-real-time/storage-events.mdx b/workflows/near-real-time/storage-events.mdx index c72ee9c..7b2759b 100644 --- a/workflows/near-real-time/storage-events.mdx +++ b/workflows/near-real-time/storage-events.mdx @@ -6,11 +6,12 @@ icon: right-to-line ## Creating a Storage Event Task -Storage Event Tasks are recurring tasks triggered when objects are created or modified in a [storage location](#storage-locations). To create a Cron task, use `tilebox.workflows.recurrent_tasks.StorageEventTask` as your tasks base class instead of the regular `tilebox.workflows.Task`. +Storage Event Tasks are automations triggered when objects are created or modified in a [storage location](#storage-locations). +To create a Storage Event task, use `tilebox.workflows.automations.StorageEventTask` as your tasks base class instead of the regular `tilebox.workflows.Task`. ```python Python from tilebox.workflows import ExecutionContext -from tilebox.workflows.recurrent_tasks import StorageEventTask, StorageEventType +from tilebox.workflows.automations import StorageEventTask, StorageEventType from tilebox.workflows.observability.logging import get_logger logger = get_logger() @@ -57,9 +58,9 @@ To list all available storage locations, use the `all` method on the storage loc from tilebox.workflows import Client client = Client() -recurrent_tasks_client = client.recurrent_tasks() +automations_client = client.automations() -storage_locations = recurrent_tasks_client.storage_locations() +storage_locations = automations_client.storage_locations() print(storage_locations) ``` @@ -112,9 +113,9 @@ After implementing a Storage Event task, register it to trigger each time a stor from tilebox.workflows import Client client = Client() -recurrent_tasks = client.recurrent_tasks() -storage_event_task = recurrent_tasks.create_recurring_storage_event_task( - "log-object-creations", # name of the recurring storage event task +automations = client.automations() +storage_event_automation = automations.create_storage_event_automation( + "log-object-creations", # name of the storage event automation "dev-cluster", # cluster slug to submit jobs to LogObjectCreation(head_bytes=20), # the task (and its input parameters) to run repeatedly triggers=[ @@ -143,7 +144,7 @@ Here are some examples of valid glob patterns: ## Start a Storage Event Task Runner -With the Storage Event task registered, a job is submitted whenever a storage event occurs. But unless a [task runner](/workflows/concepts/task-runners) is available to execute the Storage Event task the submitted jobs remain in a task queue. +With the Storage Event automation registered, a job is submitted whenever a storage event occurs. But unless a [task runner](/workflows/concepts/task-runners) is available to execute the Storage Event task the submitted jobs remain in a task queue. Once an [eligible task runner](/workflows/concepts/task-runners#task-selection) becomes available, all jobs in the queue are executed. ```python Python @@ -173,28 +174,28 @@ Inspecting the task runner output reveals that the job was submitted and the tas ## Inspecting in the Console -The [Tilebox Console](https://console.tilebox.com/workflows/recurrent-tasks) provides an easy way to inspect all registered storage event tasks. +The [Tilebox Console](https://console.tilebox.com/workflows/automations) provides an easy way to inspect all registered storage event automations. - Tilebox Workflows recurrent tasks in the Tilebox Console - Tilebox Workflows recurrent tasks in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console + Tilebox Workflows automations in the Tilebox Console -## Deleting Storage Event triggers +## Deleting Storage Event automations -To delete a registered storage event task, use `recurrent_tasks.delete`. After deletion, no new jobs will be submitted by the storage event trigger. Past jobs already triggered will still remain queued. +To delete a registered storage event automation, use `automations.delete`. After deletion, no new jobs will be submitted by the storage event trigger. Past jobs already triggered will still remain queued. ```python Python from tilebox.workflows import Client client = Client() -recurrent_tasks = client.recurrent_tasks() +automations = client.automations() -# delete the task as returned by create_recurring_storage_event_task -recurrent_tasks.delete(storage_event_task) +# delete the automation as returned by create_storage_event_automation +automations.delete(storage_event_automation) # or manually by id: -recurrent_tasks.delete("0190bafc-b3b8-88c4-008b-a5db044380d0") +automations.delete("0190bafc-b3b8-88c4-008b-a5db044380d0") ``` ## Submitting Storage Event jobs manually