Skip to content

Commit 5cee0d2

Browse files
theantagonist9509gemini-code-assist[bot]Yuan325
authored
feat(tools/dataplex-create-data-product): Add dataplex-create-data-product tool (googleapis#3504)
> [!IMPORTANT] > **Stacked Changeset:** This PR depends on changes in previous PRs (googleapis#3337, googleapis#3499, googleapis#3500, googleapis#3503). > - **Please merge previous PRs first** before reviewing/merging this one. > - The file diff below will appear large as it includes its ancestors' commits. It will resolve itself once the preceding PRs are merged and this branch is synced. This PR implements the dataplex-create-data-product tool for the Dataplex (Knowledge Catalog) source. Since creation is a Long-Running Operation (LRO), the tool immediately returns the operation ID to allow asynchronous polling. Changes overview: - Dataplex Source: Implemented CreateDataProduct which starts the creation process and returns the immediate LRO details. - New Tool: Created the dataplex-create-data-product tool exposing locationId , dataProductId , displayName , description , ownerEmails , and accessGroups parameters. - Tests: - Added unit tests for tool configuration parsing. - Added integration tests verifying LRO triggers and access restrictions. - Documentation: Created the reference page for the tool, updated the Dataplex source guide to cover LRO polling, and added it to the capabilities list. --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Yuan Teoh <45984206+Yuan325@users.noreply.github.com>
1 parent 1ddfbe9 commit 5cee0d2

15 files changed

Lines changed: 703 additions & 68 deletions

File tree

‎cmd/internal/config_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1862,7 +1862,7 @@ func TestPrebuiltTools(t *testing.T) {
18621862
},
18631863
"data-products": tools.ToolsetConfig{
18641864
Name: "data-products",
1865-
ToolNames: []string{"search_entries", "lookup_entry", "search_aspect_types", "lookup_context", "list_data_products", "get_data_product", "list_data_assets", "get_data_asset"},
1865+
ToolNames: []string{"search_entries", "lookup_entry", "search_aspect_types", "lookup_context", "list_data_products", "get_data_product", "list_data_assets", "get_data_asset", "create_data_product"},
18661866
},
18671867
"enrich": tools.ToolsetConfig{
18681868
Name: "enrich",

‎cmd/internal/imports.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ import (
160160
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataform/dataformcompilelocal"
161161
_ "github.com/googleapis/mcp-toolbox/internal/tools/datalineage/datalineagesearchlineage"
162162
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataplex/dataplexcheckdataquality"
163+
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataplex/dataplexcreatedataproduct"
163164
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataplex/dataplexdiscovermetadata"
164165
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataplex/dataplexgeneratedatainsights"
165166
_ "github.com/googleapis/mcp-toolbox/internal/tools/dataplex/dataplexgeneratedataprofile"

‎docs/KNOWLEDGE_CATALOG_README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ Once configured, the MCP server will automatically provide Knowledge Catalog cap
4646
* "Get details of the Data Product 'projects/my-project/locations/us-central1/dataProducts/my-product'."
4747
* "List Data Assets for the Data Product 'projects/my-project/locations/us-central1/dataProducts/my-product'."
4848
* "Get details of the Data Asset 'projects/my-project/locations/us-central1/dataProducts/my-product/dataAssets/my-asset'."
49+
* "Create a new Data Product named 'my-product' with owner 'user@example.com'."
4950

5051
## Server Capabilities
5152

@@ -62,6 +63,7 @@ The Knowledge Catalog MCP server provides the following tools:
6263
| `get_data_product` | Retrieve a specific Data Product. |
6364
| `list_data_assets` | List Data Assets under a Data Product. |
6465
| `get_data_asset` | Retrieve specific metadata regarding a Data Asset. |
66+
| `create_data_product` | Create a new Data Product. |
6567

6668
## Custom MCP Server Configuration
6769

‎docs/en/integrations/knowledge-catalog/prebuilt-configs/knowledge-catalog.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ aliases:
2525
* `get_data_product`: Retrieves a specific Data Product.
2626
* `list_data_assets`: Lists Data Assets under a Data Product.
2727
* `get_data_asset`: Retrieves specific metadata regarding a Data Asset.
28+
* `create_data_product`: Creates a new Data Product.
2829
* `generate_data_insights`: Creates a new Dataplex Data Documentation scan template and triggers the run.
2930
* `get_data_insights`: Retrieves the final generated data insights for a completed scan.
3031
* `generate_data_profile`: Creates a new Dataplex Data Profile scan template and triggers the run.
@@ -37,5 +38,5 @@ aliases:
3738
* `get_run_status`: Retrieves the execution status of the latest background job run.
3839
* **Toolsets:**
3940
* `discovery`: Metadata discovery and search toolset (`search_entries`, `lookup_entry`, `search_aspect_types`, `lookup_context`, `search_dq_scans`).
40-
* `data-products`: Data Products and Data Assets curation and management toolset (`search_entries`, `lookup_entry`, `search_aspect_types`, `lookup_context`, `list_data_products`, `get_data_product`, `list_data_assets`, `get_data_asset`).
41+
* `data-products`: Data Products and Data Assets curation and management toolset (`search_entries`, `lookup_entry`, `search_aspect_types`, `lookup_context`, `list_data_products`, `get_data_product`, `list_data_assets`, `get_data_asset`, `create_data_product`).
4142
* `enrich`: Metadata enrichment pipeline orchestration and execution toolset (`search_entries`, `lookup_entry`, `lookup_context`, `generate_data_insights`, `get_data_insights`, `generate_data_profile`, `get_data_profile`, `discover_metadata`, `get_discovery_results`, `check_data_quality`, `get_data_quality_results`, `get_operation`, `get_run_status`).

‎docs/en/integrations/knowledge-catalog/source.md‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,12 +395,21 @@ This abbreviated syntax works for the qualified predicates except for `label` in
395395
2. You must provide `locationId` and `dataProductId`.
396396
3. You can optionally filter the listed assets using `filter` or limit the response using `pageSize`.
397397
### Response
398-
1. Present the retrieved list of Data Assets, including their names, resources, and labels.
398+
1. Present the retrieved list of Data Assets, including their IDs, resources, and labels.
399399

400400
## Tool: get_data_asset
401401
### Request
402402
1. Use this tool to retrieve detailed metadata for a specific Data Asset.
403403
2. You must provide `locationId`, `dataProductId`, and `dataAssetId`.
404404
### Response
405-
1. Present the retrieved metadata for the Data Asset, including its name, resource, labels, and access group configurations.
405+
1. Present the retrieved metadata for the Data Asset, including its ID, resource, labels, and access group configurations.
406+
407+
## Tool: create_data_product
408+
### Request
409+
1. Use this tool to create a new Data Product.
410+
2. You must provide `locationId`. You can optionally provide `dataProductId` (if not specified, the backend will auto-generate one).
411+
3. You must provide `displayName` and `ownerEmails`.
412+
4. You can optionally provide `description` and `accessGroups`.
413+
### Response
414+
1. Present the location ID and operation ID returned immediately by the LRO creation call.
406415
```

‎docs/en/integrations/knowledge-catalog/tools/knowledge-catalog-create-data-product.md‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
title: "dataplex-create-data-product"
3+
type: docs
4+
weight: 2
5+
description: >
6+
A "dataplex-create-data-product" tool allows to create a new Data Product.
7+
---
8+
9+
## About
10+
11+
A `dataplex-create-data-product` tool creates a new Data Product in Knowledge Catalog (formerly known as Dataplex). This is a long-running operation, and the tool returns immediately with the operation's location ID and operation ID.
12+
13+
View the [Data Products guide][guide] for more information.
14+
15+
[guide]: https://docs.cloud.google.com/dataplex/docs/data-products-overview
16+
17+
## Compatible Sources
18+
19+
{{< compatible-sources >}}
20+
21+
## Requirements
22+
23+
### IAM Permissions
24+
25+
Knowledge Catalog uses [Identity and Access Management (IAM)][iam-overview] to control
26+
user and group access to Knowledge Catalog resources. Toolbox will use your
27+
[Application Default Credentials (ADC)][adc] to authorize and authenticate when
28+
interacting with [Knowledge Catalog][dataplex-docs].
29+
30+
In addition to [setting the ADC for your server][set-adc], you need to ensure
31+
the IAM identity has been given the correct IAM permissions for the tasks you
32+
intend to perform. See [Knowledge Catalog IAM permissions][iam-permissions]
33+
and [Knowledge Catalog IAM roles][iam-roles] for more information on
34+
applying IAM permissions and roles to an identity.
35+
36+
[iam-overview]: https://cloud.google.com/dataplex/docs/iam-and-access-control
37+
[adc]: https://cloud.google.com/docs/authentication#adc
38+
[set-adc]: https://cloud.google.com/docs/authentication/provide-credentials-adc
39+
[iam-permissions]: https://cloud.google.com/dataplex/docs/iam-permissions
40+
[iam-roles]: https://cloud.google.com/dataplex/docs/iam-roles
41+
[dataplex-docs]: https://cloud.google.com/dataplex
42+
43+
## Parameters
44+
45+
The `dataplex-create-data-product` tool accepts the following parameters:
46+
47+
| **field** | **type** | **required** | **description** |
48+
| ------------- | ---------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
49+
| locationId | string | true | The location ID (e.g. `us`, `us-central1`) where the Data Product should be created. |
50+
| dataProductId | string | false | The unique ID of the Data Product to create. If not specified, the backend will auto-generate a unique ID. |
51+
| displayName | string | true | The display name of the Data Product. |
52+
| description | string | false | The description of the Data Product. |
53+
| ownerEmails | array of strings | true | The list of owner emails for the Data Product. |
54+
| accessGroups | array of objects | false | List of access groups to associate with the Data Product. Each group object can contain: `id` (required), `displayName` (required), `description`, and at least one of `googleGroup` and `serviceAccount`. |
55+
56+
## Example
57+
58+
```yaml
59+
kind: tool
60+
name: create_data_product
61+
type: dataplex-create-data-product
62+
source: my-dataplex-source
63+
description: Use this tool to create a Data Product.
64+
```
65+
66+
## Reference
67+
68+
| **field** | **type** | **required** | **description** |
69+
| ----------- | -------- | ------------ | -------------------------------------------------- |
70+
| type | string | true | Must be "dataplex-create-data-product". |
71+
| source | string | true | Name of the source the tool should execute on. |
72+
| description | string | true | Description of the tool that is passed to the LLM. |

‎internal/prebuiltconfigs/tools/dataplex.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ source: dataplex-source
7272
description: Retrieves specific metadata regarding a Data Asset.
7373
---
7474
kind: tool
75+
name: create_data_product
76+
type: dataplex-create-data-product
77+
source: dataplex-source
78+
description: Creates a new Data Product.
79+
---
80+
kind: tool
7581
name: generate_data_insights
7682
type: dataplex-generate-data-insights
7783
source: dataplex-source
@@ -266,6 +272,7 @@ tools:
266272
- get_data_product
267273
- list_data_assets
268274
- get_data_asset
275+
- create_data_product
269276
---
270277
kind: toolset
271278
name: enrich

‎internal/sources/dataplex/dataplex.go‎

Lines changed: 56 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -372,9 +372,6 @@ func (s *Source) ListDataProducts(
372372
pageSize int,
373373
orderBy string,
374374
) ([]*DataProductSummary, error) {
375-
if s.GetDataProductClient() == nil {
376-
return nil, fmt.Errorf("dataplex data product client is not initialized")
377-
}
378375
if pageSize <= 0 {
379376
return nil, fmt.Errorf("pageSize must be positive: %d", pageSize)
380377
}
@@ -437,9 +434,6 @@ type DataProduct struct {
437434
}
438435

439436
func (s *Source) GetDataProduct(ctx context.Context, locationID string, dataProductID string) (*DataProduct, error) {
440-
if s.GetDataProductClient() == nil {
441-
return nil, fmt.Errorf("dataplex data product client is not initialized")
442-
}
443437
name := fmt.Sprintf("projects/%s/locations/%s/dataProducts/%s", s.ProjectID(), locationID, dataProductID)
444438
req := &dataplexpb.GetDataProductRequest{
445439
Name: name,
@@ -545,9 +539,6 @@ func (s *Source) ListDataAssets(
545539
}
546540

547541
func (s *Source) GetDataAsset(ctx context.Context, locationId string, dataProductId string, dataAssetId string) (*DataAsset, error) {
548-
if s.GetDataProductClient() == nil {
549-
return nil, fmt.Errorf("dataplex data product client is not initialized")
550-
}
551542
name := fmt.Sprintf("projects/%s/locations/%s/dataProducts/%s/dataAssets/%s", s.ProjectID(), locationId, dataProductId, dataAssetId)
552543
req := &dataplexpb.GetDataAssetRequest{
553544
Name: name,
@@ -575,6 +566,62 @@ func (s *Source) GetDataAsset(ctx context.Context, locationId string, dataProduc
575566
}, nil
576567
}
577568

569+
// CreateDataProduct creates a new Data Product.
570+
// dataProductId is optional. If empty, the Dataplex backend will automatically generate a unique ID.
571+
func (s *Source) CreateDataProduct(
572+
ctx context.Context,
573+
locationId string,
574+
dataProductId string,
575+
displayName string,
576+
description string,
577+
ownerEmails []string,
578+
accessGroups []AccessGroup,
579+
) (string, string, error) {
580+
parent := fmt.Sprintf("projects/%s/locations/%s", s.ProjectID(), locationId)
581+
582+
agMap := make(map[string]*dataplexpb.DataProduct_AccessGroup)
583+
for _, ag := range accessGroups {
584+
principal := &dataplexpb.DataProduct_Principal{}
585+
if ag.GoogleGroup != "" {
586+
principal.Type = &dataplexpb.DataProduct_Principal_GoogleGroup{
587+
GoogleGroup: ag.GoogleGroup,
588+
}
589+
}
590+
if ag.ServiceAccount != "" {
591+
principal.ServiceAccount = &ag.ServiceAccount
592+
}
593+
agMap[ag.ID] = &dataplexpb.DataProduct_AccessGroup{
594+
Id: ag.ID,
595+
DisplayName: ag.DisplayName,
596+
Description: ag.Description,
597+
Principal: principal,
598+
}
599+
}
600+
601+
req := &dataplexpb.CreateDataProductRequest{
602+
Parent: parent,
603+
DataProductId: dataProductId,
604+
DataProduct: &dataplexpb.DataProduct{
605+
DisplayName: displayName,
606+
Description: description,
607+
OwnerEmails: ownerEmails,
608+
AccessGroups: agMap,
609+
},
610+
}
611+
612+
op, err := s.GetDataProductClient().CreateDataProduct(ctx, req)
613+
if err != nil {
614+
return "", "", err
615+
}
616+
617+
opName := op.Name()
618+
parts := strings.Split(opName, "/")
619+
if len(parts) < 6 || parts[0] != "projects" || parts[2] != "locations" || parts[4] != "operations" {
620+
return "", "", fmt.Errorf("invalid operation name: %q", opName)
621+
}
622+
return parts[3], parts[5], nil
623+
}
624+
578625
func (s *Source) GenerateDataInsights(ctx context.Context, location, resourcePath string, publish bool) (string, error) {
579626
parent := fmt.Sprintf("projects/%s/locations/%s", s.ProjectID(), location)
580627
dataScanID := fmt.Sprintf("nq-doc-%s", uuid.New().String())

0 commit comments

Comments
 (0)