clusters.proto 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229
  1. // Copyright 2020 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. syntax = "proto3";
  15. package google.cloud.dataproc.v1;
  16. import "google/api/annotations.proto";
  17. import "google/api/client.proto";
  18. import "google/api/field_behavior.proto";
  19. import "google/api/resource.proto";
  20. import "google/cloud/dataproc/v1/shared.proto";
  21. import "google/longrunning/operations.proto";
  22. import "google/protobuf/duration.proto";
  23. import "google/protobuf/field_mask.proto";
  24. import "google/protobuf/timestamp.proto";
  25. option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
  26. option java_multiple_files = true;
  27. option java_outer_classname = "ClustersProto";
  28. option java_package = "com.google.cloud.dataproc.v1";
  29. option (google.api.resource_definition) = {
  30. type: "container.googleapis.com/Cluster"
  31. pattern: "projects/{project}/locations/{location}/clusters/{cluster}"
  32. };
  33. option (google.api.resource_definition) = {
  34. type: "metastore.googleapis.com/Service"
  35. pattern: "projects/{project}/locations/{location}/services/{service}"
  36. };
  37. // The ClusterControllerService provides methods to manage clusters
  38. // of Compute Engine instances.
  39. service ClusterController {
  40. option (google.api.default_host) = "dataproc.googleapis.com";
  41. option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
  42. // Creates a cluster in a project. The returned
  43. // [Operation.metadata][google.longrunning.Operation.metadata] will be
  44. // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
  45. rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
  46. option (google.api.http) = {
  47. post: "/v1/projects/{project_id}/regions/{region}/clusters"
  48. body: "cluster"
  49. };
  50. option (google.api.method_signature) = "project_id,region,cluster";
  51. option (google.longrunning.operation_info) = {
  52. response_type: "Cluster"
  53. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  54. };
  55. }
  56. // Updates a cluster in a project. The returned
  57. // [Operation.metadata][google.longrunning.Operation.metadata] will be
  58. // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
  59. rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
  60. option (google.api.http) = {
  61. patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
  62. body: "cluster"
  63. };
  64. option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
  65. option (google.longrunning.operation_info) = {
  66. response_type: "Cluster"
  67. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  68. };
  69. }
  70. // Stops a cluster in a project.
  71. rpc StopCluster(StopClusterRequest) returns (google.longrunning.Operation) {
  72. option (google.api.http) = {
  73. post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:stop"
  74. body: "*"
  75. };
  76. option (google.longrunning.operation_info) = {
  77. response_type: "Cluster"
  78. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  79. };
  80. }
  81. // Starts a cluster in a project.
  82. rpc StartCluster(StartClusterRequest) returns (google.longrunning.Operation) {
  83. option (google.api.http) = {
  84. post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:start"
  85. body: "*"
  86. };
  87. option (google.longrunning.operation_info) = {
  88. response_type: "Cluster"
  89. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  90. };
  91. }
  92. // Deletes a cluster in a project. The returned
  93. // [Operation.metadata][google.longrunning.Operation.metadata] will be
  94. // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
  95. rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
  96. option (google.api.http) = {
  97. delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
  98. };
  99. option (google.api.method_signature) = "project_id,region,cluster_name";
  100. option (google.longrunning.operation_info) = {
  101. response_type: "google.protobuf.Empty"
  102. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  103. };
  104. }
  105. // Gets the resource representation for a cluster in a project.
  106. rpc GetCluster(GetClusterRequest) returns (Cluster) {
  107. option (google.api.http) = {
  108. get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
  109. };
  110. option (google.api.method_signature) = "project_id,region,cluster_name";
  111. }
  112. // Lists all regions/{region}/clusters in a project alphabetically.
  113. rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) {
  114. option (google.api.http) = {
  115. get: "/v1/projects/{project_id}/regions/{region}/clusters"
  116. };
  117. option (google.api.method_signature) = "project_id,region";
  118. option (google.api.method_signature) = "project_id,region,filter";
  119. }
  120. // Gets cluster diagnostic information. The returned
  121. // [Operation.metadata][google.longrunning.Operation.metadata] will be
  122. // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
  123. // After the operation completes,
  124. // [Operation.response][google.longrunning.Operation.response]
  125. // contains
  126. // [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
  127. rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
  128. option (google.api.http) = {
  129. post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
  130. body: "*"
  131. };
  132. option (google.api.method_signature) = "project_id,region,cluster_name";
  133. option (google.longrunning.operation_info) = {
  134. response_type: "DiagnoseClusterResults"
  135. metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
  136. };
  137. }
  138. }
  139. // Describes the identifying information, config, and status of
  140. // a cluster of Compute Engine instances.
  141. message Cluster {
  142. // Required. The Google Cloud Platform project ID that the cluster belongs to.
  143. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  144. // Required. The cluster name. Cluster names within a project must be
  145. // unique. Names of deleted clusters can be reused.
  146. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  147. // Required. The cluster config. Note that Dataproc may set
  148. // default values, and values may change when clusters are updated.
  149. ClusterConfig config = 3 [(google.api.field_behavior) = REQUIRED];
  150. // Optional. The labels to associate with this cluster.
  151. // Label **keys** must contain 1 to 63 characters, and must conform to
  152. // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
  153. // Label **values** may be empty, but, if present, must contain 1 to 63
  154. // characters, and must conform to [RFC
  155. // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
  156. // associated with a cluster.
  157. map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL];
  158. // Output only. Cluster status.
  159. ClusterStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  160. // Output only. The previous cluster status.
  161. repeated ClusterStatus status_history = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
  162. // Output only. A cluster UUID (Unique Universal Identifier). Dataproc
  163. // generates this value when it creates the cluster.
  164. string cluster_uuid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  165. // Output only. Contains cluster daemon metrics such as HDFS and YARN stats.
  166. //
  167. // **Beta Feature**: This report is available for testing purposes only. It
  168. // may be changed before final release.
  169. ClusterMetrics metrics = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
  170. }
  171. // The cluster config.
  172. message ClusterConfig {
  173. // Optional. A Cloud Storage bucket used to stage job
  174. // dependencies, config files, and job driver console output.
  175. // If you do not specify a staging bucket, Cloud
  176. // Dataproc will determine a Cloud Storage location (US,
  177. // ASIA, or EU) for your cluster's staging bucket according to the
  178. // Compute Engine zone where your cluster is deployed, and then create
  179. // and manage this project-level, per-location bucket (see
  180. // [Dataproc staging
  181. // bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
  182. // **This field requires a Cloud Storage bucket name, not a URI to a Cloud
  183. // Storage bucket.**
  184. string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL];
  185. // Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data,
  186. // such as Spark and MapReduce history files.
  187. // If you do not specify a temp bucket,
  188. // Dataproc will determine a Cloud Storage location (US,
  189. // ASIA, or EU) for your cluster's temp bucket according to the
  190. // Compute Engine zone where your cluster is deployed, and then create
  191. // and manage this project-level, per-location bucket. The default bucket has
  192. // a TTL of 90 days, but you can use any TTL (or none) if you specify a
  193. // bucket.
  194. // **This field requires a Cloud Storage bucket name, not a URI to a Cloud
  195. // Storage bucket.**
  196. string temp_bucket = 2 [(google.api.field_behavior) = OPTIONAL];
  197. // Optional. The shared Compute Engine config settings for
  198. // all instances in a cluster.
  199. GceClusterConfig gce_cluster_config = 8 [(google.api.field_behavior) = OPTIONAL];
  200. // Optional. The Compute Engine config settings for
  201. // the master instance in a cluster.
  202. InstanceGroupConfig master_config = 9 [(google.api.field_behavior) = OPTIONAL];
  203. // Optional. The Compute Engine config settings for
  204. // worker instances in a cluster.
  205. InstanceGroupConfig worker_config = 10 [(google.api.field_behavior) = OPTIONAL];
  206. // Optional. The Compute Engine config settings for
  207. // additional worker instances in a cluster.
  208. InstanceGroupConfig secondary_worker_config = 12 [(google.api.field_behavior) = OPTIONAL];
  209. // Optional. The config settings for software inside the cluster.
  210. SoftwareConfig software_config = 13 [(google.api.field_behavior) = OPTIONAL];
  211. // Optional. Commands to execute on each node after config is
  212. // completed. By default, executables are run on master and all worker nodes.
  213. // You can test a node's `role` metadata to run an executable on
  214. // a master or worker node, as shown below using `curl` (you can also use
  215. // `wget`):
  216. //
  217. // ROLE=$(curl -H Metadata-Flavor:Google
  218. // http://metadata/computeMetadata/v1/instance/attributes/dataproc-role)
  219. // if [[ "${ROLE}" == 'Master' ]]; then
  220. // ... master specific actions ...
  221. // else
  222. // ... worker specific actions ...
  223. // fi
  224. repeated NodeInitializationAction initialization_actions = 11 [(google.api.field_behavior) = OPTIONAL];
  225. // Optional. Encryption settings for the cluster.
  226. EncryptionConfig encryption_config = 15 [(google.api.field_behavior) = OPTIONAL];
  227. // Optional. Autoscaling config for the policy associated with the cluster.
  228. // Cluster does not autoscale if this field is unset.
  229. AutoscalingConfig autoscaling_config = 18 [(google.api.field_behavior) = OPTIONAL];
  230. // Optional. Security settings for the cluster.
  231. SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];
  232. // Optional. Lifecycle setting for the cluster.
  233. LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL];
  234. // Optional. Port/endpoint configuration for this cluster
  235. EndpointConfig endpoint_config = 19 [(google.api.field_behavior) = OPTIONAL];
  236. // Optional. Metastore configuration.
  237. MetastoreConfig metastore_config = 20 [(google.api.field_behavior) = OPTIONAL];
  238. // Optional. BETA. The Kubernetes Engine config for Dataproc clusters deployed to
  239. // Kubernetes. Setting this is considered mutually exclusive with Compute
  240. // Engine-based options such as `gce_cluster_config`, `master_config`,
  241. // `worker_config`, `secondary_worker_config`, and `autoscaling_config`.
  242. GkeClusterConfig gke_cluster_config = 21 [(google.api.field_behavior) = OPTIONAL];
  243. }
  244. // The GKE config for this cluster.
  245. message GkeClusterConfig {
  246. // A full, namespace-isolated deployment target for an existing GKE cluster.
  247. message NamespacedGkeDeploymentTarget {
  248. // Optional. The target GKE cluster to deploy to.
  249. // Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'
  250. string target_gke_cluster = 1 [
  251. (google.api.field_behavior) = OPTIONAL,
  252. (google.api.resource_reference) = {
  253. type: "container.googleapis.com/Cluster"
  254. }
  255. ];
  256. // Optional. A namespace within the GKE cluster to deploy into.
  257. string cluster_namespace = 2 [(google.api.field_behavior) = OPTIONAL];
  258. }
  259. // Optional. A target for the deployment.
  260. NamespacedGkeDeploymentTarget namespaced_gke_deployment_target = 1 [(google.api.field_behavior) = OPTIONAL];
  261. }
  262. // Endpoint config for this cluster
  263. message EndpointConfig {
  264. // Output only. The map of port descriptions to URLs. Will only be populated
  265. // if enable_http_port_access is true.
  266. map<string, string> http_ports = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  267. // Optional. If true, enable http access to specific ports on the cluster
  268. // from external sources. Defaults to false.
  269. bool enable_http_port_access = 2 [(google.api.field_behavior) = OPTIONAL];
  270. }
  271. // Autoscaling Policy config associated with the cluster.
  272. message AutoscalingConfig {
  273. // Optional. The autoscaling policy used by the cluster.
  274. //
  275. // Only resource names including projectid and location (region) are valid.
  276. // Examples:
  277. //
  278. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
  279. // * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]`
  280. //
  281. // Note that the policy must be in the same project and Dataproc region.
  282. string policy_uri = 1 [(google.api.field_behavior) = OPTIONAL];
  283. }
  284. // Encryption settings for the cluster.
  285. message EncryptionConfig {
  286. // Optional. The Cloud KMS key name to use for PD disk encryption for all
  287. // instances in the cluster.
  288. string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL];
  289. }
  290. // Common config settings for resources of Compute Engine cluster
  291. // instances, applicable to all instances in the cluster.
  292. message GceClusterConfig {
  293. // `PrivateIpv6GoogleAccess` controls whether and how Dataproc cluster nodes
  294. // can communicate with Google Services through gRPC over IPv6.
  295. // These values are directly mapped to corresponding values in the
  296. // [Compute Engine Instance
  297. // fields](https://cloud.google.com/compute/docs/reference/rest/v1/instances).
  298. enum PrivateIpv6GoogleAccess {
  299. // If unspecified, Compute Engine default behavior will apply, which
  300. // is the same as [INHERIT_FROM_SUBNETWORK][google.cloud.dataproc.v1.GceClusterConfig.PrivateIpv6GoogleAccess.INHERIT_FROM_SUBNETWORK].
  301. PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED = 0;
  302. // Private access to and from Google Services configuration
  303. // inherited from the subnetwork configuration. This is the
  304. // default Compute Engine behavior.
  305. INHERIT_FROM_SUBNETWORK = 1;
  306. // Enables outbound private IPv6 access to Google Services from the Dataproc
  307. // cluster.
  308. OUTBOUND = 2;
  309. // Enables bidirectional private IPv6 access between Google Services and the
  310. // Dataproc cluster.
  311. BIDIRECTIONAL = 3;
  312. }
  313. // Optional. The zone where the Compute Engine cluster will be located.
  314. // On a create request, it is required in the "global" region. If omitted
  315. // in a non-global Dataproc region, the service will pick a zone in the
  316. // corresponding Compute Engine region. On a get request, zone will
  317. // always be present.
  318. //
  319. // A full URL, partial URI, or short name are valid. Examples:
  320. //
  321. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]`
  322. // * `projects/[project_id]/zones/[zone]`
  323. // * `us-central1-f`
  324. string zone_uri = 1 [(google.api.field_behavior) = OPTIONAL];
  325. // Optional. The Compute Engine network to be used for machine
  326. // communications. Cannot be specified with subnetwork_uri. If neither
  327. // `network_uri` nor `subnetwork_uri` is specified, the "default" network of
  328. // the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
  329. // [Using Subnetworks](https://cloud.google.com/compute/docs/subnetworks) for
  330. // more information).
  331. //
  332. // A full URL, partial URI, or short name are valid. Examples:
  333. //
  334. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default`
  335. // * `projects/[project_id]/regions/global/default`
  336. // * `default`
  337. string network_uri = 2 [(google.api.field_behavior) = OPTIONAL];
  338. // Optional. The Compute Engine subnetwork to be used for machine
  339. // communications. Cannot be specified with network_uri.
  340. //
  341. // A full URL, partial URI, or short name are valid. Examples:
  342. //
  343. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0`
  344. // * `projects/[project_id]/regions/us-east1/subnetworks/sub0`
  345. // * `sub0`
  346. string subnetwork_uri = 6 [(google.api.field_behavior) = OPTIONAL];
  347. // Optional. If true, all instances in the cluster will only have internal IP
  348. // addresses. By default, clusters are not restricted to internal IP
  349. // addresses, and will have ephemeral external IP addresses assigned to each
  350. // instance. This `internal_ip_only` restriction can only be enabled for
  351. // subnetwork enabled networks, and all off-cluster dependencies must be
  352. // configured to be accessible without external IP addresses.
  353. bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL];
  354. // Optional. The type of IPv6 access for a cluster.
  355. PrivateIpv6GoogleAccess private_ipv6_google_access = 12 [(google.api.field_behavior) = OPTIONAL];
  356. // Optional. The [Dataproc service
  357. // account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc)
  358. // (also see [VM Data Plane
  359. // identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
  360. // used by Dataproc cluster VM instances to access Google Cloud Platform
  361. // services.
  362. //
  363. // If not specified, the
  364. // [Compute Engine default service
  365. // account](https://cloud.google.com/compute/docs/access/service-accounts#default_service_account)
  366. // is used.
  367. string service_account = 8 [(google.api.field_behavior) = OPTIONAL];
  368. // Optional. The URIs of service account scopes to be included in
  369. // Compute Engine instances. The following base set of scopes is always
  370. // included:
  371. //
  372. // * https://www.googleapis.com/auth/cloud.useraccounts.readonly
  373. // * https://www.googleapis.com/auth/devstorage.read_write
  374. // * https://www.googleapis.com/auth/logging.write
  375. //
  376. // If no scopes are specified, the following defaults are also provided:
  377. //
  378. // * https://www.googleapis.com/auth/bigquery
  379. // * https://www.googleapis.com/auth/bigtable.admin.table
  380. // * https://www.googleapis.com/auth/bigtable.data
  381. // * https://www.googleapis.com/auth/devstorage.full_control
  382. repeated string service_account_scopes = 3 [(google.api.field_behavior) = OPTIONAL];
  383. // The Compute Engine tags to add to all instances (see [Tagging
  384. // instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
  385. repeated string tags = 4;
  386. // The Compute Engine metadata entries to add to all instances (see
  387. // [Project and instance
  388. // metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
  389. map<string, string> metadata = 5;
  390. // Optional. Reservation Affinity for consuming Zonal reservation.
  391. ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL];
  392. // Optional. Node Group Affinity for sole-tenant clusters.
  393. NodeGroupAffinity node_group_affinity = 13 [(google.api.field_behavior) = OPTIONAL];
  394. // Optional. Shielded Instance Config for clusters using [Compute Engine Shielded
  395. // VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm).
  396. ShieldedInstanceConfig shielded_instance_config = 14 [(google.api.field_behavior) = OPTIONAL];
  397. }
  398. // Node Group Affinity for clusters using sole-tenant node groups.
  399. message NodeGroupAffinity {
  400. // Required. The URI of a
  401. // sole-tenant [node group
  402. // resource](https://cloud.google.com/compute/docs/reference/rest/v1/nodeGroups)
  403. // that the cluster will be created on.
  404. //
  405. // A full URL, partial URI, or node group name are valid. Examples:
  406. //
  407. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1`
  408. // * `projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1`
  409. // * `node-group-1`
  410. string node_group_uri = 1 [(google.api.field_behavior) = REQUIRED];
  411. }
  412. // Shielded Instance Config for clusters using [Compute Engine Shielded
  413. // VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm).
  414. message ShieldedInstanceConfig {
  415. // Optional. Defines whether instances have Secure Boot enabled.
  416. bool enable_secure_boot = 1 [(google.api.field_behavior) = OPTIONAL];
  417. // Optional. Defines whether instances have the vTPM enabled.
  418. bool enable_vtpm = 2 [(google.api.field_behavior) = OPTIONAL];
  419. // Optional. Defines whether instances have integrity monitoring enabled.
  420. bool enable_integrity_monitoring = 3 [(google.api.field_behavior) = OPTIONAL];
  421. }
  422. // The config settings for Compute Engine resources in
  423. // an instance group, such as a master or worker group.
  424. message InstanceGroupConfig {
  425. // Controls the use of
  426. // [preemptible instances]
  427. // (https://cloud.google.com/compute/docs/instances/preemptible)
  428. // within the group.
  429. enum Preemptibility {
  430. // Preemptibility is unspecified, the system will choose the
  431. // appropriate setting for each instance group.
  432. PREEMPTIBILITY_UNSPECIFIED = 0;
  433. // Instances are non-preemptible.
  434. //
  435. // This option is allowed for all instance groups and is the only valid
  436. // value for Master and Worker instance groups.
  437. NON_PREEMPTIBLE = 1;
  438. // Instances are preemptible.
  439. //
  440. // This option is allowed only for secondary worker groups.
  441. PREEMPTIBLE = 2;
  442. }
  443. // Optional. The number of VM instances in the instance group.
  444. // For [HA
  445. // cluster](/dataproc/docs/concepts/configuring-clusters/high-availability)
  446. // [master_config](#FIELDS.master_config) groups, **must be set to 3**.
  447. // For standard cluster [master_config](#FIELDS.master_config) groups,
  448. // **must be set to 1**.
  449. int32 num_instances = 1 [(google.api.field_behavior) = OPTIONAL];
  450. // Output only. The list of instance names. Dataproc derives the names
  451. // from `cluster_name`, `num_instances`, and the instance group.
  452. repeated string instance_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  453. // Optional. The Compute Engine image resource used for cluster instances.
  454. //
  455. // The URI can represent an image or image family.
  456. //
  457. // Image examples:
  458. //
  459. // * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/[image-id]`
  460. // * `projects/[project_id]/global/images/[image-id]`
  461. // * `image-id`
  462. //
  463. // Image family examples. Dataproc will use the most recent
  464. // image from the family:
  465. //
  466. // * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/family/[custom-image-family-name]`
  467. // * `projects/[project_id]/global/images/family/[custom-image-family-name]`
  468. //
  469. // If the URI is unspecified, it will be inferred from
  470. // `SoftwareConfig.image_version` or the system default.
  471. string image_uri = 3 [(google.api.field_behavior) = OPTIONAL];
  472. // Optional. The Compute Engine machine type used for cluster instances.
  473. //
  474. // A full URL, partial URI, or short name are valid. Examples:
  475. //
  476. // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
  477. // * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
  478. // * `n1-standard-2`
  479. //
  480. // **Auto Zone Exception**: If you are using the Dataproc
  481. // [Auto Zone
  482. // Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
  483. // feature, you must use the short name of the machine type
  484. // resource, for example, `n1-standard-2`.
  485. string machine_type_uri = 4 [(google.api.field_behavior) = OPTIONAL];
  486. // Optional. Disk option config settings.
  487. DiskConfig disk_config = 5 [(google.api.field_behavior) = OPTIONAL];
  488. // Output only. Specifies that this instance group contains preemptible
  489. // instances.
  490. bool is_preemptible = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  491. // Optional. Specifies the preemptibility of the instance group.
  492. //
  493. // The default value for master and worker groups is
  494. // `NON_PREEMPTIBLE`. This default cannot be changed.
  495. //
  496. // The default value for secondary instances is
  497. // `PREEMPTIBLE`.
  498. Preemptibility preemptibility = 10 [(google.api.field_behavior) = OPTIONAL];
  499. // Output only. The config for Compute Engine Instance Group
  500. // Manager that manages this group.
  501. // This is only used for preemptible instance groups.
  502. ManagedGroupConfig managed_group_config = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
  503. // Optional. The Compute Engine accelerator configuration for these
  504. // instances.
  505. repeated AcceleratorConfig accelerators = 8 [(google.api.field_behavior) = OPTIONAL];
  506. // Optional. Specifies the minimum cpu platform for the Instance Group.
  507. // See [Dataproc -> Minimum CPU
  508. // Platform](https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).
  509. string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL];
  510. }
  511. // Specifies the resources used to actively manage an instance group.
  512. message ManagedGroupConfig {
  513. // Output only. The name of the Instance Template used for the Managed
  514. // Instance Group.
  515. string instance_template_name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  516. // Output only. The name of the Instance Group Manager for this group.
  517. string instance_group_manager_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  518. }
  519. // Specifies the type and number of accelerator cards attached to the instances
  520. // of an instance. See [GPUs on Compute
  521. // Engine](https://cloud.google.com/compute/docs/gpus/).
  522. message AcceleratorConfig {
  523. // Full URL, partial URI, or short name of the accelerator type resource to
  524. // expose to this instance. See
  525. // [Compute Engine
  526. // AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes).
  527. //
  528. // Examples:
  529. //
  530. // * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
  531. // * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
  532. // * `nvidia-tesla-k80`
  533. //
  534. // **Auto Zone Exception**: If you are using the Dataproc
  535. // [Auto Zone
  536. // Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
  537. // feature, you must use the short name of the accelerator type
  538. // resource, for example, `nvidia-tesla-k80`.
  539. string accelerator_type_uri = 1;
  540. // The number of the accelerator cards of this type exposed to this instance.
  541. int32 accelerator_count = 2;
  542. }
  543. // Specifies the config of disk options for a group of VM instances.
  544. message DiskConfig {
  545. // Optional. Type of the boot disk (default is "pd-standard").
  546. // Valid values: "pd-balanced" (Persistent Disk Balanced Solid State Drive),
  547. // "pd-ssd" (Persistent Disk Solid State Drive),
  548. // or "pd-standard" (Persistent Disk Hard Disk Drive).
  549. // See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types).
  550. string boot_disk_type = 3 [(google.api.field_behavior) = OPTIONAL];
  551. // Optional. Size in GB of the boot disk (default is 500GB).
  552. int32 boot_disk_size_gb = 1 [(google.api.field_behavior) = OPTIONAL];
  553. // Optional. Number of attached SSDs, from 0 to 4 (default is 0).
  554. // If SSDs are not attached, the boot disk is used to store runtime logs and
  555. // [HDFS](https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data.
  556. // If one or more SSDs are attached, this runtime bulk
  557. // data is spread across them, and the boot disk contains only basic
  558. // config and installed binaries.
  559. int32 num_local_ssds = 2 [(google.api.field_behavior) = OPTIONAL];
  560. }
  561. // Specifies an executable to run on a fully configured node and a
  562. // timeout period for executable completion.
  563. message NodeInitializationAction {
  564. // Required. Cloud Storage URI of executable file.
  565. string executable_file = 1 [(google.api.field_behavior) = REQUIRED];
  566. // Optional. Amount of time executable has to complete. Default is
  567. // 10 minutes (see JSON representation of
  568. // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  569. //
  570. // Cluster creation fails with an explanatory error message (the
  571. // name of the executable that caused the error and the exceeded timeout
  572. // period) if the executable is not completed at end of the timeout period.
  573. google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
  574. }
  575. // The status of a cluster and its instances.
  576. message ClusterStatus {
  577. // The cluster state.
  578. enum State {
  579. // The cluster state is unknown.
  580. UNKNOWN = 0;
  581. // The cluster is being created and set up. It is not ready for use.
  582. CREATING = 1;
  583. // The cluster is currently running and healthy. It is ready for use.
  584. RUNNING = 2;
  585. // The cluster encountered an error. It is not ready for use.
  586. ERROR = 3;
  587. // The cluster is being deleted. It cannot be used.
  588. DELETING = 4;
  589. // The cluster is being updated. It continues to accept and process jobs.
  590. UPDATING = 5;
  591. // The cluster is being stopped. It cannot be used.
  592. STOPPING = 6;
  593. // The cluster is currently stopped. It is not ready for use.
  594. STOPPED = 7;
  595. // The cluster is being started. It is not ready for use.
  596. STARTING = 8;
  597. }
  598. // The cluster substate.
  599. enum Substate {
  600. // The cluster substate is unknown.
  601. UNSPECIFIED = 0;
  602. // The cluster is known to be in an unhealthy state
  603. // (for example, critical daemons are not running or HDFS capacity is
  604. // exhausted).
  605. //
  606. // Applies to RUNNING state.
  607. UNHEALTHY = 1;
  608. // The agent-reported status is out of date (may occur if
  609. // Dataproc loses communication with Agent).
  610. //
  611. // Applies to RUNNING state.
  612. STALE_STATUS = 2;
  613. }
  614. // Output only. The cluster's state.
  615. State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  616. // Optional. Output only. Details of cluster's state.
  617. string detail = 2 [
  618. (google.api.field_behavior) = OUTPUT_ONLY,
  619. (google.api.field_behavior) = OPTIONAL
  620. ];
  621. // Output only. Time when this state was entered (see JSON representation of
  622. // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  623. google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  624. // Output only. Additional state information that includes
  625. // status reported by the agent.
  626. Substate substate = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  627. }
  628. // Security related configuration, including encryption, Kerberos, etc.
  629. message SecurityConfig {
  630. // Optional. Kerberos related configuration.
  631. KerberosConfig kerberos_config = 1 [(google.api.field_behavior) = OPTIONAL];
  632. // Optional. Identity related configuration, including service account based
  633. // secure multi-tenancy user mappings.
  634. IdentityConfig identity_config = 2 [(google.api.field_behavior) = OPTIONAL];
  635. }
  636. // Specifies Kerberos related configuration.
  637. message KerberosConfig {
  638. // Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set
  639. // this field to true to enable Kerberos on a cluster.
  640. bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL];
  641. // Optional. The Cloud Storage URI of a KMS encrypted file containing the root
  642. // principal password.
  643. string root_principal_password_uri = 2 [(google.api.field_behavior) = OPTIONAL];
  644. // Optional. The uri of the KMS key used to encrypt various sensitive
  645. // files.
  646. string kms_key_uri = 3 [(google.api.field_behavior) = OPTIONAL];
  647. // Optional. The Cloud Storage URI of the keystore file used for SSL
  648. // encryption. If not provided, Dataproc will provide a self-signed
  649. // certificate.
  650. string keystore_uri = 4 [(google.api.field_behavior) = OPTIONAL];
  651. // Optional. The Cloud Storage URI of the truststore file used for SSL
  652. // encryption. If not provided, Dataproc will provide a self-signed
  653. // certificate.
  654. string truststore_uri = 5 [(google.api.field_behavior) = OPTIONAL];
  655. // Optional. The Cloud Storage URI of a KMS encrypted file containing the
  656. // password to the user provided keystore. For the self-signed certificate,
  657. // this password is generated by Dataproc.
  658. string keystore_password_uri = 6 [(google.api.field_behavior) = OPTIONAL];
  659. // Optional. The Cloud Storage URI of a KMS encrypted file containing the
  660. // password to the user provided key. For the self-signed certificate, this
  661. // password is generated by Dataproc.
  662. string key_password_uri = 7 [(google.api.field_behavior) = OPTIONAL];
  663. // Optional. The Cloud Storage URI of a KMS encrypted file containing the
  664. // password to the user provided truststore. For the self-signed certificate,
  665. // this password is generated by Dataproc.
  666. string truststore_password_uri = 8 [(google.api.field_behavior) = OPTIONAL];
  667. // Optional. The remote realm the Dataproc on-cluster KDC will trust, should
  668. // the user enable cross realm trust.
  669. string cross_realm_trust_realm = 9 [(google.api.field_behavior) = OPTIONAL];
  670. // Optional. The KDC (IP or hostname) for the remote trusted realm in a cross
  671. // realm trust relationship.
  672. string cross_realm_trust_kdc = 10 [(google.api.field_behavior) = OPTIONAL];
  673. // Optional. The admin server (IP or hostname) for the remote trusted realm in
  674. // a cross realm trust relationship.
  675. string cross_realm_trust_admin_server = 11 [(google.api.field_behavior) = OPTIONAL];
  676. // Optional. The Cloud Storage URI of a KMS encrypted file containing the
  677. // shared password between the on-cluster Kerberos realm and the remote
  678. // trusted realm, in a cross realm trust relationship.
  679. string cross_realm_trust_shared_password_uri = 12 [(google.api.field_behavior) = OPTIONAL];
  680. // Optional. The Cloud Storage URI of a KMS encrypted file containing the
  681. // master key of the KDC database.
  682. string kdc_db_key_uri = 13 [(google.api.field_behavior) = OPTIONAL];
  683. // Optional. The lifetime of the ticket granting ticket, in hours.
  684. // If not specified, or user specifies 0, then default value 10
  685. // will be used.
  686. int32 tgt_lifetime_hours = 14 [(google.api.field_behavior) = OPTIONAL];
  687. // Optional. The name of the on-cluster Kerberos realm.
  688. // If not specified, the uppercased domain of hostnames will be the realm.
  689. string realm = 15 [(google.api.field_behavior) = OPTIONAL];
  690. }
  691. // Identity related configuration, including service account based
  692. // secure multi-tenancy user mappings.
  693. message IdentityConfig {
  694. // Required. Map of user to service account.
  695. map<string, string> user_service_account_mapping = 1 [(google.api.field_behavior) = REQUIRED];
  696. }
  697. // Specifies the selection and config of software inside the cluster.
  698. message SoftwareConfig {
  699. // Optional. The version of software inside the cluster. It must be one of the
  700. // supported [Dataproc
  701. // Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
  702. // such as "1.2" (including a subminor version, such as "1.2.29"), or the
  703. // ["preview"
  704. // version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
  705. // If unspecified, it defaults to the latest Debian version.
  706. string image_version = 1 [(google.api.field_behavior) = OPTIONAL];
  707. // Optional. The properties to set on daemon config files.
  708. //
  709. // Property keys are specified in `prefix:property` format, for example
  710. // `core:hadoop.tmp.dir`. The following are supported prefixes
  711. // and their mappings:
  712. //
  713. // * capacity-scheduler: `capacity-scheduler.xml`
  714. // * core: `core-site.xml`
  715. // * distcp: `distcp-default.xml`
  716. // * hdfs: `hdfs-site.xml`
  717. // * hive: `hive-site.xml`
  718. // * mapred: `mapred-site.xml`
  719. // * pig: `pig.properties`
  720. // * spark: `spark-defaults.conf`
  721. // * yarn: `yarn-site.xml`
  722. //
  723. // For more information, see [Cluster
  724. // properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
  725. map<string, string> properties = 2 [(google.api.field_behavior) = OPTIONAL];
  726. // Optional. The set of components to activate on the cluster.
  727. repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
  728. }
  729. // Specifies the cluster auto-delete schedule configuration.
  730. message LifecycleConfig {
  731. // Optional. The duration to keep the cluster alive while idling (when no jobs
  732. // are running). Passing this threshold will cause the cluster to be
  733. // deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON
  734. // representation of
  735. // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  736. google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL];
  737. // Either the exact time the cluster should be deleted at or
  738. // the cluster maximum age.
  739. oneof ttl {
  740. // Optional. The time when cluster will be auto-deleted (see JSON representation of
  741. // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  742. google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL];
  743. // Optional. The lifetime duration of cluster. The cluster will be
  744. // auto-deleted at the end of this period. Minimum value is 10 minutes;
  745. // maximum value is 14 days (see JSON representation of
  746. // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  747. google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL];
  748. }
  749. // Output only. The time when cluster became idle (most recent job finished)
  750. // and became eligible for deletion due to idleness (see JSON representation
  751. // of
  752. // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  753. google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  754. }
  755. // Specifies a Metastore configuration.
  756. message MetastoreConfig {
  757. // Required. Resource name of an existing Dataproc Metastore service.
  758. //
  759. // Example:
  760. //
  761. // * `projects/[project_id]/locations/[dataproc_region]/services/[service-name]`
  762. string dataproc_metastore_service = 1 [
  763. (google.api.field_behavior) = REQUIRED,
  764. (google.api.resource_reference) = {
  765. type: "metastore.googleapis.com/Service"
  766. }
  767. ];
  768. }
  769. // Contains cluster daemon metrics, such as HDFS and YARN stats.
  770. //
  771. // **Beta Feature**: This report is available for testing purposes only. It may
  772. // be changed before final release.
  773. message ClusterMetrics {
  774. // The HDFS metrics.
  775. map<string, int64> hdfs_metrics = 1;
  776. // The YARN metrics.
  777. map<string, int64> yarn_metrics = 2;
  778. }
  779. // A request to create a cluster.
  780. message CreateClusterRequest {
  781. // Required. The ID of the Google Cloud Platform project that the cluster
  782. // belongs to.
  783. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  784. // Required. The Dataproc region in which to handle the request.
  785. string region = 3 [(google.api.field_behavior) = REQUIRED];
  786. // Required. The cluster to create.
  787. Cluster cluster = 2 [(google.api.field_behavior) = REQUIRED];
  788. // Optional. A unique id used to identify the request. If the server receives two
  789. // [CreateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateClusterRequest)s
  790. // with the same id, then the second request will be ignored and the
  791. // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the backend
  792. // is returned.
  793. //
  794. // It is recommended to always set this value to a
  795. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  796. //
  797. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  798. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  799. string request_id = 4 [(google.api.field_behavior) = OPTIONAL];
  800. }
  801. // A request to update a cluster.
  802. message UpdateClusterRequest {
  803. // Required. The ID of the Google Cloud Platform project the
  804. // cluster belongs to.
  805. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  806. // Required. The Dataproc region in which to handle the request.
  807. string region = 5 [(google.api.field_behavior) = REQUIRED];
  808. // Required. The cluster name.
  809. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  810. // Required. The changes to the cluster.
  811. Cluster cluster = 3 [(google.api.field_behavior) = REQUIRED];
  812. // Optional. Timeout for graceful YARN decomissioning. Graceful
  813. // decommissioning allows removing nodes from the cluster without
  814. // interrupting jobs in progress. Timeout specifies how long to wait for jobs
  815. // in progress to finish before forcefully removing nodes (and potentially
  816. // interrupting jobs). Default timeout is 0 (for forceful decommission), and
  817. // the maximum allowed timeout is 1 day. (see JSON representation of
  818. // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  819. //
  820. // Only supported on Dataproc image versions 1.2 and higher.
  821. google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
  822. // Required. Specifies the path, relative to `Cluster`, of
  823. // the field to update. For example, to change the number of workers
  824. // in a cluster to 5, the `update_mask` parameter would be
  825. // specified as `config.worker_config.num_instances`,
  826. // and the `PATCH` request body would specify the new value, as follows:
  827. //
  828. // {
  829. // "config":{
  830. // "workerConfig":{
  831. // "numInstances":"5"
  832. // }
  833. // }
  834. // }
  835. // Similarly, to change the number of preemptible workers in a cluster to 5,
  836. // the `update_mask` parameter would be
  837. // `config.secondary_worker_config.num_instances`, and the `PATCH` request
  838. // body would be set as follows:
  839. //
  840. // {
  841. // "config":{
  842. // "secondaryWorkerConfig":{
  843. // "numInstances":"5"
  844. // }
  845. // }
  846. // }
  847. // <strong>Note:</strong> Currently, only the following fields can be updated:
  848. //
  849. // <table>
  850. // <tbody>
  851. // <tr>
  852. // <td><strong>Mask</strong></td>
  853. // <td><strong>Purpose</strong></td>
  854. // </tr>
  855. // <tr>
  856. // <td><strong><em>labels</em></strong></td>
  857. // <td>Update labels</td>
  858. // </tr>
  859. // <tr>
  860. // <td><strong><em>config.worker_config.num_instances</em></strong></td>
  861. // <td>Resize primary worker group</td>
  862. // </tr>
  863. // <tr>
  864. // <td><strong><em>config.secondary_worker_config.num_instances</em></strong></td>
  865. // <td>Resize secondary worker group</td>
  866. // </tr>
  867. // <tr>
  868. // <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or
  869. // change autoscaling policies</td>
  870. // </tr>
  871. // </tbody>
  872. // </table>
  873. google.protobuf.FieldMask update_mask = 4 [(google.api.field_behavior) = REQUIRED];
  874. // Optional. A unique id used to identify the request. If the server
  875. // receives two
  876. // [UpdateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.UpdateClusterRequest)s
  877. // with the same id, then the second request will be ignored and the
  878. // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
  879. // backend is returned.
  880. //
  881. // It is recommended to always set this value to a
  882. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  883. //
  884. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  885. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  886. string request_id = 7 [(google.api.field_behavior) = OPTIONAL];
  887. }
  888. // A request to stop a cluster.
  889. message StopClusterRequest {
  890. // Required. The ID of the Google Cloud Platform project the
  891. // cluster belongs to.
  892. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  893. // Required. The Dataproc region in which to handle the request.
  894. string region = 2 [(google.api.field_behavior) = REQUIRED];
  895. // Required. The cluster name.
  896. string cluster_name = 3 [(google.api.field_behavior) = REQUIRED];
  897. // Optional. Specifying the `cluster_uuid` means the RPC will fail
  898. // (with error NOT_FOUND) if a cluster with the specified UUID does not exist.
  899. string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL];
  900. // Optional. A unique id used to identify the request. If the server
  901. // receives two
  902. // [StopClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StopClusterRequest)s
  903. // with the same id, then the second request will be ignored and the
  904. // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
  905. // backend is returned.
  906. //
  907. // Recommendation: Set this value to a
  908. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  909. //
  910. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  911. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  912. string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
  913. }
  914. // A request to start a cluster.
  915. message StartClusterRequest {
  916. // Required. The ID of the Google Cloud Platform project the
  917. // cluster belongs to.
  918. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  919. // Required. The Dataproc region in which to handle the request.
  920. string region = 2 [(google.api.field_behavior) = REQUIRED];
  921. // Required. The cluster name.
  922. string cluster_name = 3 [(google.api.field_behavior) = REQUIRED];
  923. // Optional. Specifying the `cluster_uuid` means the RPC will fail
  924. // (with error NOT_FOUND) if a cluster with the specified UUID does not exist.
  925. string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL];
  926. // Optional. A unique id used to identify the request. If the server
  927. // receives two
  928. // [StartClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StartClusterRequest)s
  929. // with the same id, then the second request will be ignored and the
  930. // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
  931. // backend is returned.
  932. //
  933. // Recommendation: Set this value to a
  934. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  935. //
  936. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  937. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  938. string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
  939. }
  940. // A request to delete a cluster.
  941. message DeleteClusterRequest {
  942. // Required. The ID of the Google Cloud Platform project that the cluster
  943. // belongs to.
  944. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  945. // Required. The Dataproc region in which to handle the request.
  946. string region = 3 [(google.api.field_behavior) = REQUIRED];
  947. // Required. The cluster name.
  948. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  949. // Optional. Specifying the `cluster_uuid` means the RPC should fail
  950. // (with error NOT_FOUND) if cluster with specified UUID does not exist.
  951. string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL];
  952. // Optional. A unique id used to identify the request. If the server
  953. // receives two
  954. // [DeleteClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.DeleteClusterRequest)s
  955. // with the same id, then the second request will be ignored and the
  956. // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
  957. // backend is returned.
  958. //
  959. // It is recommended to always set this value to a
  960. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  961. //
  962. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  963. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  964. string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
  965. }
  966. // Request to get the resource representation for a cluster in a project.
  967. message GetClusterRequest {
  968. // Required. The ID of the Google Cloud Platform project that the cluster
  969. // belongs to.
  970. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  971. // Required. The Dataproc region in which to handle the request.
  972. string region = 3 [(google.api.field_behavior) = REQUIRED];
  973. // Required. The cluster name.
  974. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  975. }
  976. // A request to list the clusters in a project.
  977. message ListClustersRequest {
  978. // Required. The ID of the Google Cloud Platform project that the cluster
  979. // belongs to.
  980. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  981. // Required. The Dataproc region in which to handle the request.
  982. string region = 4 [(google.api.field_behavior) = REQUIRED];
  983. // Optional. A filter constraining the clusters to list. Filters are
  984. // case-sensitive and have the following syntax:
  985. //
  986. // field = value [AND [field = value]] ...
  987. //
  988. // where **field** is one of `status.state`, `clusterName`, or `labels.[KEY]`,
  989. // and `[KEY]` is a label key. **value** can be `*` to match all values.
  990. // `status.state` can be one of the following: `ACTIVE`, `INACTIVE`,
  991. // `CREATING`, `RUNNING`, `ERROR`, `DELETING`, or `UPDATING`. `ACTIVE`
  992. // contains the `CREATING`, `UPDATING`, and `RUNNING` states. `INACTIVE`
  993. // contains the `DELETING` and `ERROR` states.
  994. // `clusterName` is the name of the cluster provided at creation time.
  995. // Only the logical `AND` operator is supported; space-separated items are
  996. // treated as having an implicit `AND` operator.
  997. //
  998. // Example filter:
  999. //
  1000. // status.state = ACTIVE AND clusterName = mycluster
  1001. // AND labels.env = staging AND labels.starred = *
  1002. string filter = 5 [(google.api.field_behavior) = OPTIONAL];
  1003. // Optional. The standard List page size.
  1004. int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
  1005. // Optional. The standard List page token.
  1006. string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
  1007. }
  1008. // The list of all clusters in a project.
  1009. message ListClustersResponse {
  1010. // Output only. The clusters in the project.
  1011. repeated Cluster clusters = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  1012. // Output only. This token is included in the response if there are more
  1013. // results to fetch. To fetch additional results, provide this value as the
  1014. // `page_token` in a subsequent `ListClustersRequest`.
  1015. string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  1016. }
  1017. // A request to collect cluster diagnostic information.
  1018. message DiagnoseClusterRequest {
  1019. // Required. The ID of the Google Cloud Platform project that the cluster
  1020. // belongs to.
  1021. string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  1022. // Required. The Dataproc region in which to handle the request.
  1023. string region = 3 [(google.api.field_behavior) = REQUIRED];
  1024. // Required. The cluster name.
  1025. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  1026. }
  1027. // The location of diagnostic output.
  1028. message DiagnoseClusterResults {
  1029. // Output only. The Cloud Storage URI of the diagnostic output.
  1030. // The output report is a plain text file with a summary of collected
  1031. // diagnostics.
  1032. string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  1033. }
  1034. // Reservation Affinity for consuming Zonal reservation.
  1035. message ReservationAffinity {
  1036. // Indicates whether to consume capacity from an reservation or not.
  1037. enum Type {
  1038. TYPE_UNSPECIFIED = 0;
  1039. // Do not consume from any allocated capacity.
  1040. NO_RESERVATION = 1;
  1041. // Consume any reservation available.
  1042. ANY_RESERVATION = 2;
  1043. // Must consume from a specific reservation. Must specify key value fields
  1044. // for specifying the reservations.
  1045. SPECIFIC_RESERVATION = 3;
  1046. }
  1047. // Optional. Type of reservation to consume
  1048. Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL];
  1049. // Optional. Corresponds to the label key of reservation resource.
  1050. string key = 2 [(google.api.field_behavior) = OPTIONAL];
  1051. // Optional. Corresponds to the label values of reservation resource.
  1052. repeated string values = 3 [(google.api.field_behavior) = OPTIONAL];
  1053. }