dataset.proto 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. syntax = "proto3";
  15. package google.cloud.aiplatform.v1;
  16. import "google/api/field_behavior.proto";
  17. import "google/api/resource.proto";
  18. import "google/cloud/aiplatform/v1/encryption_spec.proto";
  19. import "google/cloud/aiplatform/v1/io.proto";
  20. import "google/protobuf/struct.proto";
  21. import "google/protobuf/timestamp.proto";
  22. import "google/api/annotations.proto";
  23. option csharp_namespace = "Google.Cloud.AIPlatform.V1";
  24. option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
  25. option java_multiple_files = true;
  26. option java_outer_classname = "DatasetProto";
  27. option java_package = "com.google.cloud.aiplatform.v1";
  28. option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
  29. option ruby_package = "Google::Cloud::AIPlatform::V1";
  30. // A collection of DataItems and Annotations on them.
  31. message Dataset {
  32. option (google.api.resource) = {
  33. type: "aiplatform.googleapis.com/Dataset"
  34. pattern: "projects/{project}/locations/{location}/datasets/{dataset}"
  35. };
  36. // Output only. The resource name of the Dataset.
  37. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  38. // Required. The user-defined name of the Dataset.
  39. // The name can be up to 128 characters long and can be consist of any UTF-8
  40. // characters.
  41. string display_name = 2 [(google.api.field_behavior) = REQUIRED];
  42. // Required. Points to a YAML file stored on Google Cloud Storage describing additional
  43. // information about the Dataset.
  44. // The schema is defined as an OpenAPI 3.0.2 Schema Object.
  45. // The schema files that can be used here are found in
  46. // gs://google-cloud-aiplatform/schema/dataset/metadata/.
  47. string metadata_schema_uri = 3 [(google.api.field_behavior) = REQUIRED];
  48. // Required. Additional information about the Dataset.
  49. google.protobuf.Value metadata = 8 [(google.api.field_behavior) = REQUIRED];
  50. // Output only. Timestamp when this Dataset was created.
  51. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  52. // Output only. Timestamp when this Dataset was last updated.
  53. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  54. // Used to perform consistent read-modify-write updates. If not set, a blind
  55. // "overwrite" update happens.
  56. string etag = 6;
  57. // The labels with user-defined metadata to organize your Datasets.
  58. //
  59. // Label keys and values can be no longer than 64 characters
  60. // (Unicode codepoints), can only contain lowercase letters, numeric
  61. // characters, underscores and dashes. International characters are allowed.
  62. // No more than 64 user labels can be associated with one Dataset (System
  63. // labels are excluded).
  64. //
  65. // See https://goo.gl/xmQnxf for more information and examples of labels.
  66. // System reserved label keys are prefixed with "aiplatform.googleapis.com/"
  67. // and are immutable. Following system labels exist for each Dataset:
  68. //
  69. // * "aiplatform.googleapis.com/dataset_metadata_schema": output only, its
  70. // value is the [metadata_schema's][google.cloud.aiplatform.v1.Dataset.metadata_schema_uri] title.
  71. map<string, string> labels = 7;
  72. // Customer-managed encryption key spec for a Dataset. If set, this Dataset
  73. // and all sub-resources of this Dataset will be secured by this key.
  74. EncryptionSpec encryption_spec = 11;
  75. }
  76. // Describes the location from where we import data into a Dataset, together
  77. // with the labels that will be applied to the DataItems and the Annotations.
  78. message ImportDataConfig {
  79. // The source of the input.
  80. oneof source {
  81. // The Google Cloud Storage location for the input content.
  82. GcsSource gcs_source = 1;
  83. }
  84. // Labels that will be applied to newly imported DataItems. If an identical
  85. // DataItem as one being imported already exists in the Dataset, then these
  86. // labels will be appended to these of the already existing one, and if labels
  87. // with identical key is imported before, the old label value will be
  88. // overwritten. If two DataItems are identical in the same import data
  89. // operation, the labels will be combined and if key collision happens in this
  90. // case, one of the values will be picked randomly. Two DataItems are
  91. // considered identical if their content bytes are identical (e.g. image bytes
  92. // or pdf bytes).
  93. // These labels will be overridden by Annotation labels specified inside index
  94. // file referenced by [import_schema_uri][google.cloud.aiplatform.v1.ImportDataConfig.import_schema_uri], e.g. jsonl file.
  95. map<string, string> data_item_labels = 2;
  96. // Required. Points to a YAML file stored on Google Cloud Storage describing the import
  97. // format. Validation will be done against the schema. The schema is defined
  98. // as an [OpenAPI 3.0.2 Schema
  99. // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
  100. string import_schema_uri = 4 [(google.api.field_behavior) = REQUIRED];
  101. }
  102. // Describes what part of the Dataset is to be exported, the destination of
  103. // the export and how to export.
  104. message ExportDataConfig {
  105. // The destination of the output.
  106. oneof destination {
  107. // The Google Cloud Storage location where the output is to be written to.
  108. // In the given directory a new directory will be created with name:
  109. // `export-data-<dataset-display-name>-<timestamp-of-export-call>` where
  110. // timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. All export
  111. // output will be written into that directory. Inside that directory,
  112. // annotations with the same schema will be grouped into sub directories
  113. // which are named with the corresponding annotations' schema title. Inside
  114. // these sub directories, a schema.yaml will be created to describe the
  115. // output format.
  116. GcsDestination gcs_destination = 1;
  117. }
  118. // A filter on Annotations of the Dataset. Only Annotations on to-be-exported
  119. // DataItems(specified by [data_items_filter][]) that match this filter will
  120. // be exported. The filter syntax is the same as in
  121. // [ListAnnotations][google.cloud.aiplatform.v1.DatasetService.ListAnnotations].
  122. string annotations_filter = 2;
  123. }