feat(tables): add partition_by and sorted_by configuration

.openapi-generator/FILES

@@ -117,9 +117,11 @@ docs/SecretMetadataResponse.md
 docs/SecretsApi.md
 docs/SubmitJobResponse.md
 docs/TableInfo.md
+docs/TablePartitionKey.md
 docs/TableProfileResponse.md
 docs/TableRefreshError.md
 docs/TableRefreshResult.md
+docs/TableSortKey.md
 docs/TemporalProfileDetail.md
 docs/TextProfileDetail.md
 docs/UpdateEmbeddingProviderRequest.md
@@ -262,9 +264,11 @@ hotdata/models/schema_refresh_result.py
 hotdata/models/secret_metadata_response.py
 hotdata/models/submit_job_response.py
 hotdata/models/table_info.py
+hotdata/models/table_partition_key.py
 hotdata/models/table_profile_response.py
 hotdata/models/table_refresh_error.py
 hotdata/models/table_refresh_result.py
+hotdata/models/table_sort_key.py
 hotdata/models/temporal_profile_detail.py
 hotdata/models/text_profile_detail.py
 hotdata/models/update_embedding_provider_request.py
@@ -398,9 +402,11 @@ test/test_secret_metadata_response.py
 test/test_secrets_api.py
 test/test_submit_job_response.py
 test/test_table_info.py
+test/test_table_partition_key.py
 test/test_table_profile_response.py
 test/test_table_refresh_error.py
 test/test_table_refresh_result.py
+test/test_table_sort_key.py
 test/test_temporal_profile_detail.py
 test/test_text_profile_detail.py
 test/test_update_embedding_provider_request.py

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- feat(tables): add partition_by and sorted_by configuration
 - chore(query-runs): clarify user_public_id documentation
 - feat(databases): add search parameter to list endpoint
 - chore(databases): make pagination fields nullable

docs/AddManagedTableDecl.md

@@ -8,6 +8,8 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **key** | **List[str]** | Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected. | [optional] 
 **name** | **str** |  | 
+**partition_by** | [**List[TablePartitionKey]**](TablePartitionKey.md) | Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter. | [optional] 
+**sorted_by** | [**List[TableSortKey]**](TableSortKey.md) | Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter. | [optional] 
 
 ## Example
 

docs/AddManagedTableRequest.md

@@ -8,6 +8,8 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **key** | **List[str]** | Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected. | [optional] 
 **name** | **str** |  | 
+**partition_by** | [**List[TablePartitionKey]**](TablePartitionKey.md) | Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter. | [optional] 
+**sorted_by** | [**List[TableSortKey]**](TableSortKey.md) | Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter. | [optional] 
 
 ## Example
 

docs/DatabaseDefaultTableDecl.md

@@ -8,6 +8,8 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **key** | **List[str]** | Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected. | [optional] 
 **name** | **str** |  | 
+**partition_by** | [**List[TablePartitionKey]**](TablePartitionKey.md) | Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter. | [optional] 
+**sorted_by** | [**List[TableSortKey]**](TableSortKey.md) | Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter. | [optional] 
 
 ## Example
 

docs/TablePartitionKey.md

@@ -0,0 +1,31 @@
+# TablePartitionKey
+
+One partition key of a table's storage layout.  Partitioning groups rows that share a key value into their own files, so a query filtering on that key reads only the matching files. Keys are applied in the order given, and several keys may read the same column: to get one partition per calendar month, declare `year` and `month` on the timestamp column. A single calendar transform on its own is rarely what you want — `month` alone puts every March of every year in one partition.
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**column** | **str** | Column the key reads. | 
+**transform** | **str** | How the value is derived from the column. One of `identity` (the column value itself), `year`, `month`, `day`, or `hour`. | 
+
+## Example
+
+```python
+from hotdata.models.table_partition_key import TablePartitionKey
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of TablePartitionKey from a JSON string
+table_partition_key_instance = TablePartitionKey.from_json(json)
+# print the JSON string representation of the object
+print(TablePartitionKey.to_json())
+
+# convert the object into a dict
+table_partition_key_dict = table_partition_key_instance.to_dict()
+# create an instance of TablePartitionKey from a dict
+table_partition_key_from_dict = TablePartitionKey.from_dict(table_partition_key_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

docs/TableSortKey.md

@@ -0,0 +1,32 @@
+# TableSortKey
+
+One key of a table's sort order.  Rows are written in this order, which keeps the values in each file within a narrow range and lets queries filtering on those columns skip files entirely. Most useful on columns you filter by ranges, such as a timestamp.
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**column** | **str** |  | 
+**direction** | **str** | `asc` (the default) or `desc`. | [optional] 
+**nulls** | **str** | Where nulls are placed: `first` or `last`. Defaults to the SQL default for the chosen direction. | [optional] 
+
+## Example
+
+```python
+from hotdata.models.table_sort_key import TableSortKey
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of TableSortKey from a JSON string
+table_sort_key_instance = TableSortKey.from_json(json)
+# print the JSON string representation of the object
+print(TableSortKey.to_json())
+
+# convert the object into a dict
+table_sort_key_dict = table_sort_key_instance.to_dict()
+# create an instance of TableSortKey from a dict
+table_sort_key_from_dict = TableSortKey.from_dict(table_sort_key_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

hotdata/__init__.py

@@ -152,9 +152,11 @@
     "SecretMetadataResponse",
     "SubmitJobResponse",
     "TableInfo",
+    "TablePartitionKey",
     "TableProfileResponse",
     "TableRefreshError",
     "TableRefreshResult",
+    "TableSortKey",
     "TemporalProfileDetail",
     "TextProfileDetail",
     "UpdateEmbeddingProviderRequest",
@@ -303,9 +305,11 @@
 from hotdata.models.secret_metadata_response import SecretMetadataResponse as SecretMetadataResponse
 from hotdata.models.submit_job_response import SubmitJobResponse as SubmitJobResponse
 from hotdata.models.table_info import TableInfo as TableInfo
+from hotdata.models.table_partition_key import TablePartitionKey as TablePartitionKey
 from hotdata.models.table_profile_response import TableProfileResponse as TableProfileResponse
 from hotdata.models.table_refresh_error import TableRefreshError as TableRefreshError
 from hotdata.models.table_refresh_result import TableRefreshResult as TableRefreshResult
+from hotdata.models.table_sort_key import TableSortKey as TableSortKey
 from hotdata.models.temporal_profile_detail import TemporalProfileDetail as TemporalProfileDetail
 from hotdata.models.text_profile_detail import TextProfileDetail as TextProfileDetail
 from hotdata.models.update_embedding_provider_request import UpdateEmbeddingProviderRequest as UpdateEmbeddingProviderRequest

hotdata/models/__init__.py

@@ -116,9 +116,11 @@
 from hotdata.models.secret_metadata_response import SecretMetadataResponse
 from hotdata.models.submit_job_response import SubmitJobResponse
 from hotdata.models.table_info import TableInfo
+from hotdata.models.table_partition_key import TablePartitionKey
 from hotdata.models.table_profile_response import TableProfileResponse
 from hotdata.models.table_refresh_error import TableRefreshError
 from hotdata.models.table_refresh_result import TableRefreshResult
+from hotdata.models.table_sort_key import TableSortKey
 from hotdata.models.temporal_profile_detail import TemporalProfileDetail
 from hotdata.models.text_profile_detail import TextProfileDetail
 from hotdata.models.update_embedding_provider_request import UpdateEmbeddingProviderRequest

hotdata/models/add_managed_table_decl.py

@@ -20,6 +20,8 @@
 
 from pydantic import BaseModel, ConfigDict, Field, StrictStr
 from typing import Any, ClassVar, Dict, List, Optional
+from hotdata.models.table_partition_key import TablePartitionKey
+from hotdata.models.table_sort_key import TableSortKey
 from typing import Optional, Set
 from typing_extensions import Self
 
@@ -29,7 +31,9 @@ class AddManagedTableDecl(BaseModel):
     """ # noqa: E501
     key: Optional[List[StrictStr]] = Field(default=None, description="Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected.")
     name: StrictStr
-    __properties: ClassVar[List[str]] = ["key", "name"]
+    partition_by: Optional[List[TablePartitionKey]] = Field(default=None, description="Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter.")
+    sorted_by: Optional[List[TableSortKey]] = Field(default=None, description="Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter.")
+    __properties: ClassVar[List[str]] = ["key", "name", "partition_by", "sorted_by"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -70,6 +74,20 @@ def to_dict(self) -> Dict[str, Any]:
             exclude=excluded_fields,
             exclude_none=True,
         )
+        # override the default output from pydantic by calling `to_dict()` of each item in partition_by (list)
+        _items = []
+        if self.partition_by:
+            for _item_partition_by in self.partition_by:
+                if _item_partition_by:
+                    _items.append(_item_partition_by.to_dict())
+            _dict['partition_by'] = _items
+        # override the default output from pydantic by calling `to_dict()` of each item in sorted_by (list)
+        _items = []
+        if self.sorted_by:
+            for _item_sorted_by in self.sorted_by:
+                if _item_sorted_by:
+                    _items.append(_item_sorted_by.to_dict())
+            _dict['sorted_by'] = _items
         return _dict
 
     @classmethod
@@ -83,7 +101,9 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
 
         _obj = cls.model_validate({
             "key": obj.get("key"),
-            "name": obj.get("name")
+            "name": obj.get("name"),
+            "partition_by": [TablePartitionKey.from_dict(_item) for _item in obj["partition_by"]] if obj.get("partition_by") is not None else None,
+            "sorted_by": [TableSortKey.from_dict(_item) for _item in obj["sorted_by"]] if obj.get("sorted_by") is not None else None
         })
         return _obj
 

hotdata/models/add_managed_table_request.py

@@ -20,6 +20,8 @@
 
 from pydantic import BaseModel, ConfigDict, Field, StrictStr
 from typing import Any, ClassVar, Dict, List, Optional
+from hotdata.models.table_partition_key import TablePartitionKey
+from hotdata.models.table_sort_key import TableSortKey
 from typing import Optional, Set
 from typing_extensions import Self
 
@@ -29,7 +31,9 @@ class AddManagedTableRequest(BaseModel):
     """ # noqa: E501
     key: Optional[List[StrictStr]] = Field(default=None, description="Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected.")
     name: StrictStr
-    __properties: ClassVar[List[str]] = ["key", "name"]
+    partition_by: Optional[List[TablePartitionKey]] = Field(default=None, description="Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter.")
+    sorted_by: Optional[List[TableSortKey]] = Field(default=None, description="Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter.")
+    __properties: ClassVar[List[str]] = ["key", "name", "partition_by", "sorted_by"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -70,6 +74,20 @@ def to_dict(self) -> Dict[str, Any]:
             exclude=excluded_fields,
             exclude_none=True,
         )
+        # override the default output from pydantic by calling `to_dict()` of each item in partition_by (list)
+        _items = []
+        if self.partition_by:
+            for _item_partition_by in self.partition_by:
+                if _item_partition_by:
+                    _items.append(_item_partition_by.to_dict())
+            _dict['partition_by'] = _items
+        # override the default output from pydantic by calling `to_dict()` of each item in sorted_by (list)
+        _items = []
+        if self.sorted_by:
+            for _item_sorted_by in self.sorted_by:
+                if _item_sorted_by:
+                    _items.append(_item_sorted_by.to_dict())
+            _dict['sorted_by'] = _items
         return _dict
 
     @classmethod
@@ -83,7 +101,9 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
 
         _obj = cls.model_validate({
             "key": obj.get("key"),
-            "name": obj.get("name")
+            "name": obj.get("name"),
+            "partition_by": [TablePartitionKey.from_dict(_item) for _item in obj["partition_by"]] if obj.get("partition_by") is not None else None,
+            "sorted_by": [TableSortKey.from_dict(_item) for _item in obj["sorted_by"]] if obj.get("sorted_by") is not None else None
         })
         return _obj
 

hotdata/models/database_default_table_decl.py

@@ -20,6 +20,8 @@
 
 from pydantic import BaseModel, ConfigDict, Field, StrictStr
 from typing import Any, ClassVar, Dict, List, Optional
+from hotdata.models.table_partition_key import TablePartitionKey
+from hotdata.models.table_sort_key import TableSortKey
 from typing import Optional, Set
 from typing_extensions import Self
 
@@ -29,7 +31,9 @@ class DatabaseDefaultTableDecl(BaseModel):
     """ # noqa: E501
     key: Optional[List[StrictStr]] = Field(default=None, description="Columns that uniquely identify a row, enabling the key-based load modes (`delete`, `update`, `upsert`) on this table: those loads match rows by these columns' values. Omit (the default) to declare no key; the table can still be loaded with `replace` and `append`, but key-based modes are then rejected.")
     name: StrictStr
-    __properties: ClassVar[List[str]] = ["key", "name"]
+    partition_by: Optional[List[TablePartitionKey]] = Field(default=None, description="Partition keys for this table, applied in order. Omit for no partitioning. Declared when the table is created and fixed thereafter.")
+    sorted_by: Optional[List[TableSortKey]] = Field(default=None, description="Sort keys for this table, applied in order. Omit for no sort order. Declared when the table is created and fixed thereafter.")
+    __properties: ClassVar[List[str]] = ["key", "name", "partition_by", "sorted_by"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -70,6 +74,20 @@ def to_dict(self) -> Dict[str, Any]:
             exclude=excluded_fields,
             exclude_none=True,
         )
+        # override the default output from pydantic by calling `to_dict()` of each item in partition_by (list)
+        _items = []
+        if self.partition_by:
+            for _item_partition_by in self.partition_by:
+                if _item_partition_by:
+                    _items.append(_item_partition_by.to_dict())
+            _dict['partition_by'] = _items
+        # override the default output from pydantic by calling `to_dict()` of each item in sorted_by (list)
+        _items = []
+        if self.sorted_by:
+            for _item_sorted_by in self.sorted_by:
+                if _item_sorted_by:
+                    _items.append(_item_sorted_by.to_dict())
+            _dict['sorted_by'] = _items
         return _dict
 
     @classmethod
@@ -83,7 +101,9 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
 
         _obj = cls.model_validate({
             "key": obj.get("key"),
-            "name": obj.get("name")
+            "name": obj.get("name"),
+            "partition_by": [TablePartitionKey.from_dict(_item) for _item in obj["partition_by"]] if obj.get("partition_by") is not None else None,
+            "sorted_by": [TableSortKey.from_dict(_item) for _item in obj["sorted_by"]] if obj.get("sorted_by") is not None else None
         })
         return _obj