workflow_templates.proto 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836
  1. // Copyright 2020 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. syntax = "proto3";
  15. package google.cloud.dataproc.v1;
  16. import "google/api/annotations.proto";
  17. import "google/api/client.proto";
  18. import "google/api/field_behavior.proto";
  19. import "google/api/resource.proto";
  20. import "google/cloud/dataproc/v1/clusters.proto";
  21. import "google/cloud/dataproc/v1/jobs.proto";
  22. import "google/longrunning/operations.proto";
  23. import "google/protobuf/duration.proto";
  24. import "google/protobuf/empty.proto";
  25. import "google/protobuf/timestamp.proto";
  26. option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
  27. option java_multiple_files = true;
  28. option java_outer_classname = "WorkflowTemplatesProto";
  29. option java_package = "com.google.cloud.dataproc.v1";
  30. // The API interface for managing Workflow Templates in the
  31. // Dataproc API.
  32. service WorkflowTemplateService {
  33. option (google.api.default_host) = "dataproc.googleapis.com";
  34. option (google.api.oauth_scopes) =
  35. "https://www.googleapis.com/auth/cloud-platform";
  36. // Creates new workflow template.
  37. rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest)
  38. returns (WorkflowTemplate) {
  39. option (google.api.http) = {
  40. post: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
  41. body: "template"
  42. additional_bindings {
  43. post: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
  44. body: "template"
  45. }
  46. };
  47. option (google.api.method_signature) = "parent,template";
  48. }
  49. // Retrieves the latest workflow template.
  50. //
  51. // Can retrieve previously instantiated template by specifying optional
  52. // version parameter.
  53. rpc GetWorkflowTemplate(GetWorkflowTemplateRequest)
  54. returns (WorkflowTemplate) {
  55. option (google.api.http) = {
  56. get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
  57. additional_bindings {
  58. get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
  59. }
  60. };
  61. option (google.api.method_signature) = "name";
  62. }
  63. // Instantiates a template and begins execution.
  64. //
  65. // The returned Operation can be used to track execution of
  66. // workflow by polling
  67. // [operations.get][google.longrunning.Operations.GetOperation].
  68. // The Operation will complete when entire workflow is finished.
  69. //
  70. // The running workflow can be aborted via
  71. // [operations.cancel][google.longrunning.Operations.CancelOperation].
  72. // This will cause any inflight jobs to be cancelled and workflow-owned
  73. // clusters to be deleted.
  74. //
  75. // The [Operation.metadata][google.longrunning.Operation.metadata] will be
  76. // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata).
  77. // Also see [Using
  78. // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
  79. //
  80. // On successful completion,
  81. // [Operation.response][google.longrunning.Operation.response] will be
  82. // [Empty][google.protobuf.Empty].
  83. rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest)
  84. returns (google.longrunning.Operation) {
  85. option (google.api.http) = {
  86. post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate"
  87. body: "*"
  88. additional_bindings {
  89. post: "/v1/{name=projects/*/regions/*/workflowTemplates/*}:instantiate"
  90. body: "*"
  91. }
  92. };
  93. option (google.api.method_signature) = "name";
  94. option (google.api.method_signature) = "name,parameters";
  95. option (google.longrunning.operation_info) = {
  96. response_type: "google.protobuf.Empty"
  97. metadata_type: "WorkflowMetadata"
  98. };
  99. }
  100. // Instantiates a template and begins execution.
  101. //
  102. // This method is equivalent to executing the sequence
  103. // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate],
  104. // [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate],
  105. // [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate].
  106. //
  107. // The returned Operation can be used to track execution of
  108. // workflow by polling
  109. // [operations.get][google.longrunning.Operations.GetOperation].
  110. // The Operation will complete when entire workflow is finished.
  111. //
  112. // The running workflow can be aborted via
  113. // [operations.cancel][google.longrunning.Operations.CancelOperation].
  114. // This will cause any inflight jobs to be cancelled and workflow-owned
  115. // clusters to be deleted.
  116. //
  117. // The [Operation.metadata][google.longrunning.Operation.metadata] will be
  118. // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata).
  119. // Also see [Using
  120. // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
  121. //
  122. // On successful completion,
  123. // [Operation.response][google.longrunning.Operation.response] will be
  124. // [Empty][google.protobuf.Empty].
  125. rpc InstantiateInlineWorkflowTemplate(
  126. InstantiateInlineWorkflowTemplateRequest)
  127. returns (google.longrunning.Operation) {
  128. option (google.api.http) = {
  129. post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline"
  130. body: "template"
  131. additional_bindings {
  132. post: "/v1/{parent=projects/*/regions/*}/workflowTemplates:instantiateInline"
  133. body: "template"
  134. }
  135. };
  136. option (google.api.method_signature) = "parent,template";
  137. option (google.longrunning.operation_info) = {
  138. response_type: "google.protobuf.Empty"
  139. metadata_type: "WorkflowMetadata"
  140. };
  141. }
  142. // Updates (replaces) workflow template. The updated template
  143. // must contain version that matches the current server version.
  144. rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest)
  145. returns (WorkflowTemplate) {
  146. option (google.api.http) = {
  147. put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}"
  148. body: "template"
  149. additional_bindings {
  150. put: "/v1/{template.name=projects/*/regions/*/workflowTemplates/*}"
  151. body: "template"
  152. }
  153. };
  154. option (google.api.method_signature) = "template";
  155. }
  156. // Lists workflows that match the specified filter in the request.
  157. rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest)
  158. returns (ListWorkflowTemplatesResponse) {
  159. option (google.api.http) = {
  160. get: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
  161. additional_bindings {
  162. get: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
  163. }
  164. };
  165. option (google.api.method_signature) = "parent";
  166. }
  167. // Deletes a workflow template. It does not cancel in-progress workflows.
  168. rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest)
  169. returns (google.protobuf.Empty) {
  170. option (google.api.http) = {
  171. delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
  172. additional_bindings {
  173. delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
  174. }
  175. };
  176. option (google.api.method_signature) = "name";
  177. }
  178. }
  179. // A Dataproc workflow template resource.
  180. message WorkflowTemplate {
  181. option (google.api.resource) = {
  182. type: "dataproc.googleapis.com/WorkflowTemplate"
  183. pattern: "projects/{project}/regions/{region}/workflowTemplates/{workflow_template}"
  184. pattern: "projects/{project}/locations/{location}/workflowTemplates/{workflow_template}"
  185. history: ORIGINALLY_SINGLE_PATTERN
  186. };
  187. string id = 2 [(google.api.field_behavior) = REQUIRED];
  188. // Output only. The resource name of the workflow template, as described
  189. // in https://cloud.google.com/apis/design/resource_names.
  190. //
  191. // * For `projects.regions.workflowTemplates`, the resource name of the
  192. // template has the following format:
  193. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  194. //
  195. // * For `projects.locations.workflowTemplates`, the resource name of the
  196. // template has the following format:
  197. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  198. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  199. // Optional. Used to perform a consistent read-modify-write.
  200. //
  201. // This field should be left blank for a `CreateWorkflowTemplate` request. It
  202. // is required for an `UpdateWorkflowTemplate` request, and must match the
  203. // current server version. A typical update template flow would fetch the
  204. // current template with a `GetWorkflowTemplate` request, which will return
  205. // the current template with the `version` field filled in with the
  206. // current server version. The user updates other fields in the template,
  207. // then returns it as part of the `UpdateWorkflowTemplate` request.
  208. int32 version = 3 [(google.api.field_behavior) = OPTIONAL];
  209. // Output only. The time template was created.
  210. google.protobuf.Timestamp create_time = 4
  211. [(google.api.field_behavior) = OUTPUT_ONLY];
  212. // Output only. The time template was last updated.
  213. google.protobuf.Timestamp update_time = 5
  214. [(google.api.field_behavior) = OUTPUT_ONLY];
  215. // Optional. The labels to associate with this template. These labels
  216. // will be propagated to all jobs and clusters created by the workflow
  217. // instance.
  218. //
  219. // Label **keys** must contain 1 to 63 characters, and must conform to
  220. // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
  221. //
  222. // Label **values** may be empty, but, if present, must contain 1 to 63
  223. // characters, and must conform to
  224. // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
  225. //
  226. // No more than 32 labels can be associated with a template.
  227. map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
  228. // Required. WorkflowTemplate scheduling information.
  229. WorkflowTemplatePlacement placement = 7
  230. [(google.api.field_behavior) = REQUIRED];
  231. // Required. The Directed Acyclic Graph of Jobs to submit.
  232. repeated OrderedJob jobs = 8 [(google.api.field_behavior) = REQUIRED];
  233. // Optional. Template parameters whose values are substituted into the
  234. // template. Values for parameters must be provided when the template is
  235. // instantiated.
  236. repeated TemplateParameter parameters = 9
  237. [(google.api.field_behavior) = OPTIONAL];
  238. // Optional. Timeout duration for the DAG of jobs, expressed in seconds (see
  239. // [JSON representation of
  240. // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  241. // The timeout duration must be from 10 minutes ("600s") to 24 hours
  242. // ("86400s"). The timer begins when the first job is submitted. If the
  243. // workflow is running at the end of the timeout period, any remaining jobs
  244. // are cancelled, the workflow is ended, and if the workflow was running on a
  245. // [managed
  246. // cluster](/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
  247. // the cluster is deleted.
  248. google.protobuf.Duration dag_timeout = 10
  249. [(google.api.field_behavior) = OPTIONAL];
  250. }
  251. // Specifies workflow execution target.
  252. //
  253. // Either `managed_cluster` or `cluster_selector` is required.
  254. message WorkflowTemplatePlacement {
  255. // Required. Specifies where workflow executes; either on a managed
  256. // cluster or an existing cluster chosen by labels.
  257. oneof placement {
  258. // A cluster that is managed by the workflow.
  259. ManagedCluster managed_cluster = 1;
  260. // Optional. A selector that chooses target cluster for jobs based
  261. // on metadata.
  262. //
  263. // The selector is evaluated at the time each job is submitted.
  264. ClusterSelector cluster_selector = 2;
  265. }
  266. }
  267. // Cluster that is managed by the workflow.
  268. message ManagedCluster {
  269. // Required. The cluster name prefix. A unique cluster name will be formed by
  270. // appending a random suffix.
  271. //
  272. // The name must contain only lower-case letters (a-z), numbers (0-9),
  273. // and hyphens (-). Must begin with a letter. Cannot begin or end with
  274. // hyphen. Must consist of between 2 and 35 characters.
  275. string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];
  276. // Required. The cluster configuration.
  277. ClusterConfig config = 3 [(google.api.field_behavior) = REQUIRED];
  278. // Optional. The labels to associate with this cluster.
  279. //
  280. // Label keys must be between 1 and 63 characters long, and must conform to
  281. // the following PCRE regular expression:
  282. // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
  283. //
  284. // Label values must be between 1 and 63 characters long, and must conform to
  285. // the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
  286. //
  287. // No more than 32 labels can be associated with a given cluster.
  288. map<string, string> labels = 4 [(google.api.field_behavior) = OPTIONAL];
  289. }
  290. // A selector that chooses target cluster for jobs based on metadata.
  291. message ClusterSelector {
  292. // Optional. The zone where workflow process executes. This parameter does not
  293. // affect the selection of the cluster.
  294. //
  295. // If unspecified, the zone of the first cluster matching the selector
  296. // is used.
  297. string zone = 1 [(google.api.field_behavior) = OPTIONAL];
  298. // Required. The cluster labels. Cluster must have all labels
  299. // to match.
  300. map<string, string> cluster_labels = 2
  301. [(google.api.field_behavior) = REQUIRED];
  302. }
  303. // A job executed by the workflow.
  304. message OrderedJob {
  305. // Required. The step id. The id must be unique among all jobs
  306. // within the template.
  307. //
  308. // The step id is used as prefix for job id, as job
  309. // `goog-dataproc-workflow-step-id` label, and in
  310. // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids]
  311. // field from other steps.
  312. //
  313. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  314. // underscores (_), and hyphens (-). Cannot begin or end with underscore
  315. // or hyphen. Must consist of between 3 and 50 characters.
  316. string step_id = 1 [(google.api.field_behavior) = REQUIRED];
  317. // Required. The job definition.
  318. oneof job_type {
  319. // Optional. Job is a Hadoop job.
  320. HadoopJob hadoop_job = 2 [(google.api.field_behavior) = OPTIONAL];
  321. // Optional. Job is a Spark job.
  322. SparkJob spark_job = 3 [(google.api.field_behavior) = OPTIONAL];
  323. // Optional. Job is a PySpark job.
  324. PySparkJob pyspark_job = 4 [(google.api.field_behavior) = OPTIONAL];
  325. // Optional. Job is a Hive job.
  326. HiveJob hive_job = 5 [(google.api.field_behavior) = OPTIONAL];
  327. // Optional. Job is a Pig job.
  328. PigJob pig_job = 6 [(google.api.field_behavior) = OPTIONAL];
  329. // Optional. Job is a SparkR job.
  330. SparkRJob spark_r_job = 11 [(google.api.field_behavior) = OPTIONAL];
  331. // Optional. Job is a SparkSql job.
  332. SparkSqlJob spark_sql_job = 7 [(google.api.field_behavior) = OPTIONAL];
  333. // Optional. Job is a Presto job.
  334. PrestoJob presto_job = 12 [(google.api.field_behavior) = OPTIONAL];
  335. }
  336. // Optional. The labels to associate with this job.
  337. //
  338. // Label keys must be between 1 and 63 characters long, and must conform to
  339. // the following regular expression:
  340. // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
  341. //
  342. // Label values must be between 1 and 63 characters long, and must conform to
  343. // the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
  344. //
  345. // No more than 32 labels can be associated with a given job.
  346. map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL];
  347. // Optional. Job scheduling configuration.
  348. JobScheduling scheduling = 9 [(google.api.field_behavior) = OPTIONAL];
  349. // Optional. The optional list of prerequisite job step_ids.
  350. // If not specified, the job will start at the beginning of workflow.
  351. repeated string prerequisite_step_ids = 10
  352. [(google.api.field_behavior) = OPTIONAL];
  353. }
  354. // A configurable parameter that replaces one or more fields in the template.
  355. // Parameterizable fields:
  356. // - Labels
  357. // - File uris
  358. // - Job properties
  359. // - Job arguments
  360. // - Script variables
  361. // - Main class (in HadoopJob and SparkJob)
  362. // - Zone (in ClusterSelector)
  363. message TemplateParameter {
  364. // Required. Parameter name.
  365. // The parameter name is used as the key, and paired with the
  366. // parameter value, which are passed to the template when the template
  367. // is instantiated.
  368. // The name must contain only capital letters (A-Z), numbers (0-9), and
  369. // underscores (_), and must not start with a number. The maximum length is
  370. // 40 characters.
  371. string name = 1 [(google.api.field_behavior) = REQUIRED];
  372. // Required. Paths to all fields that the parameter replaces.
  373. // A field is allowed to appear in at most one parameter's list of field
  374. // paths.
  375. //
  376. // A field path is similar in syntax to a
  377. // [google.protobuf.FieldMask][google.protobuf.FieldMask]. For example, a
  378. // field path that references the zone field of a workflow template's cluster
  379. // selector would be specified as `placement.clusterSelector.zone`.
  380. //
  381. // Also, field paths can reference fields using the following syntax:
  382. //
  383. // * Values in maps can be referenced by key:
  384. // * labels['key']
  385. // * placement.clusterSelector.clusterLabels['key']
  386. // * placement.managedCluster.labels['key']
  387. // * placement.clusterSelector.clusterLabels['key']
  388. // * jobs['step-id'].labels['key']
  389. //
  390. // * Jobs in the jobs list can be referenced by step-id:
  391. // * jobs['step-id'].hadoopJob.mainJarFileUri
  392. // * jobs['step-id'].hiveJob.queryFileUri
  393. // * jobs['step-id'].pySparkJob.mainPythonFileUri
  394. // * jobs['step-id'].hadoopJob.jarFileUris[0]
  395. // * jobs['step-id'].hadoopJob.archiveUris[0]
  396. // * jobs['step-id'].hadoopJob.fileUris[0]
  397. // * jobs['step-id'].pySparkJob.pythonFileUris[0]
  398. //
  399. // * Items in repeated fields can be referenced by a zero-based index:
  400. // * jobs['step-id'].sparkJob.args[0]
  401. //
  402. // * Other examples:
  403. // * jobs['step-id'].hadoopJob.properties['key']
  404. // * jobs['step-id'].hadoopJob.args[0]
  405. // * jobs['step-id'].hiveJob.scriptVariables['key']
  406. // * jobs['step-id'].hadoopJob.mainJarFileUri
  407. // * placement.clusterSelector.zone
  408. //
  409. // It may not be possible to parameterize maps and repeated fields in their
  410. // entirety since only individual map values and individual items in repeated
  411. // fields can be referenced. For example, the following field paths are
  412. // invalid:
  413. //
  414. // - placement.clusterSelector.clusterLabels
  415. // - jobs['step-id'].sparkJob.args
  416. repeated string fields = 2 [(google.api.field_behavior) = REQUIRED];
  417. // Optional. Brief description of the parameter.
  418. // Must not exceed 1024 characters.
  419. string description = 3 [(google.api.field_behavior) = OPTIONAL];
  420. // Optional. Validation rules to be applied to this parameter's value.
  421. ParameterValidation validation = 4 [(google.api.field_behavior) = OPTIONAL];
  422. }
  423. // Configuration for parameter validation.
  424. message ParameterValidation {
  425. // Required. The type of validation to be performed.
  426. oneof validation_type {
  427. // Validation based on regular expressions.
  428. RegexValidation regex = 1;
  429. // Validation based on a list of allowed values.
  430. ValueValidation values = 2;
  431. }
  432. }
  433. // Validation based on regular expressions.
  434. message RegexValidation {
  435. // Required. RE2 regular expressions used to validate the parameter's value.
  436. // The value must match the regex in its entirety (substring
  437. // matches are not sufficient).
  438. repeated string regexes = 1 [(google.api.field_behavior) = REQUIRED];
  439. }
  440. // Validation based on a list of allowed values.
  441. message ValueValidation {
  442. // Required. List of allowed values for the parameter.
  443. repeated string values = 1 [(google.api.field_behavior) = REQUIRED];
  444. }
  445. // A Dataproc workflow template resource.
  446. message WorkflowMetadata {
  447. // The operation state.
  448. enum State {
  449. // Unused.
  450. UNKNOWN = 0;
  451. // The operation has been created.
  452. PENDING = 1;
  453. // The operation is running.
  454. RUNNING = 2;
  455. // The operation is done; either cancelled or completed.
  456. DONE = 3;
  457. }
  458. // Output only. The resource name of the workflow template as described
  459. // in https://cloud.google.com/apis/design/resource_names.
  460. //
  461. // * For `projects.regions.workflowTemplates`, the resource name of the
  462. // template has the following format:
  463. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  464. //
  465. // * For `projects.locations.workflowTemplates`, the resource name of the
  466. // template has the following format:
  467. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  468. string template = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  469. // Output only. The version of template at the time of
  470. // workflow instantiation.
  471. int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  472. // Output only. The create cluster operation metadata.
  473. ClusterOperation create_cluster = 3
  474. [(google.api.field_behavior) = OUTPUT_ONLY];
  475. // Output only. The workflow graph.
  476. WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  477. // Output only. The delete cluster operation metadata.
  478. ClusterOperation delete_cluster = 5
  479. [(google.api.field_behavior) = OUTPUT_ONLY];
  480. // Output only. The workflow state.
  481. State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  482. // Output only. The name of the target cluster.
  483. string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
  484. // Map from parameter names to values that were used for those parameters.
  485. map<string, string> parameters = 8;
  486. // Output only. Workflow start time.
  487. google.protobuf.Timestamp start_time = 9
  488. [(google.api.field_behavior) = OUTPUT_ONLY];
  489. // Output only. Workflow end time.
  490. google.protobuf.Timestamp end_time = 10
  491. [(google.api.field_behavior) = OUTPUT_ONLY];
  492. // Output only. The UUID of target cluster.
  493. string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
  494. // Output only. The timeout duration for the DAG of jobs, expressed in seconds
  495. // (see [JSON representation of
  496. // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
  497. google.protobuf.Duration dag_timeout = 12
  498. [(google.api.field_behavior) = OUTPUT_ONLY];
  499. // Output only. DAG start time, only set for workflows with
  500. // [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when
  501. // DAG begins.
  502. google.protobuf.Timestamp dag_start_time = 13
  503. [(google.api.field_behavior) = OUTPUT_ONLY];
  504. // Output only. DAG end time, only set for workflows with
  505. // [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when
  506. // DAG ends.
  507. google.protobuf.Timestamp dag_end_time = 14
  508. [(google.api.field_behavior) = OUTPUT_ONLY];
  509. }
  510. // The cluster operation triggered by a workflow.
  511. message ClusterOperation {
  512. // Output only. The id of the cluster operation.
  513. string operation_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  514. // Output only. Error, if operation failed.
  515. string error = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  516. // Output only. Indicates the operation is done.
  517. bool done = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  518. }
  519. // The workflow graph.
  520. message WorkflowGraph {
  521. // Output only. The workflow nodes.
  522. repeated WorkflowNode nodes = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  523. }
  524. // The workflow node.
  525. message WorkflowNode {
  526. // The workflow node state.
  527. enum NodeState {
  528. // State is unspecified.
  529. NODE_STATE_UNSPECIFIED = 0;
  530. // The node is awaiting prerequisite node to finish.
  531. BLOCKED = 1;
  532. // The node is runnable but not running.
  533. RUNNABLE = 2;
  534. // The node is running.
  535. RUNNING = 3;
  536. // The node completed successfully.
  537. COMPLETED = 4;
  538. // The node failed. A node can be marked FAILED because
  539. // its ancestor or peer failed.
  540. FAILED = 5;
  541. }
  542. // Output only. The name of the node.
  543. string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  544. // Output only. Node's prerequisite nodes.
  545. repeated string prerequisite_step_ids = 2
  546. [(google.api.field_behavior) = OUTPUT_ONLY];
  547. // Output only. The job id; populated after the node enters RUNNING state.
  548. string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  549. // Output only. The node state.
  550. NodeState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  551. // Output only. The error detail.
  552. string error = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  553. }
  554. // A request to create a workflow template.
  555. message CreateWorkflowTemplateRequest {
  556. // Required. The resource name of the region or location, as described
  557. // in https://cloud.google.com/apis/design/resource_names.
  558. //
  559. // * For `projects.regions.workflowTemplates,create`, the resource name of the
  560. // region has the following format:
  561. // `projects/{project_id}/regions/{region}`
  562. //
  563. // * For `projects.locations.workflowTemplates.create`, the resource name of
  564. // the location has the following format:
  565. // `projects/{project_id}/locations/{location}`
  566. string parent = 1 [
  567. (google.api.field_behavior) = REQUIRED,
  568. (google.api.resource_reference) = {
  569. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  570. }
  571. ];
  572. // Required. The Dataproc workflow template to create.
  573. WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
  574. }
  575. // A request to fetch a workflow template.
  576. message GetWorkflowTemplateRequest {
  577. // Required. The resource name of the workflow template, as described
  578. // in https://cloud.google.com/apis/design/resource_names.
  579. //
  580. // * For `projects.regions.workflowTemplates.get`, the resource name of the
  581. // template has the following format:
  582. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  583. //
  584. // * For `projects.locations.workflowTemplates.get`, the resource name of the
  585. // template has the following format:
  586. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  587. string name = 1 [
  588. (google.api.field_behavior) = REQUIRED,
  589. (google.api.resource_reference) = {
  590. type: "dataproc.googleapis.com/WorkflowTemplate"
  591. }
  592. ];
  593. // Optional. The version of workflow template to retrieve. Only previously
  594. // instantiated versions can be retrieved.
  595. //
  596. // If unspecified, retrieves the current version.
  597. int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
  598. }
  599. // A request to instantiate a workflow template.
  600. message InstantiateWorkflowTemplateRequest {
  601. // Required. The resource name of the workflow template, as described
  602. // in https://cloud.google.com/apis/design/resource_names.
  603. //
  604. // * For `projects.regions.workflowTemplates.instantiate`, the resource name
  605. // of the template has the following format:
  606. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  607. //
  608. // * For `projects.locations.workflowTemplates.instantiate`, the resource name
  609. // of the template has the following format:
  610. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  611. string name = 1 [
  612. (google.api.field_behavior) = REQUIRED,
  613. (google.api.resource_reference) = {
  614. type: "dataproc.googleapis.com/WorkflowTemplate"
  615. }
  616. ];
  617. // Optional. The version of workflow template to instantiate. If specified,
  618. // the workflow will be instantiated only if the current version of
  619. // the workflow template has the supplied version.
  620. //
  621. // This option cannot be used to instantiate a previous version of
  622. // workflow template.
  623. int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
  624. // Optional. A tag that prevents multiple concurrent workflow
  625. // instances with the same tag from running. This mitigates risk of
  626. // concurrent instances started due to retries.
  627. //
  628. // It is recommended to always set this value to a
  629. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  630. //
  631. // The tag must contain only letters (a-z, A-Z), numbers (0-9),
  632. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  633. string request_id = 5 [(google.api.field_behavior) = OPTIONAL];
  634. // Optional. Map from parameter names to values that should be used for those
  635. // parameters. Values may not exceed 1000 characters.
  636. map<string, string> parameters = 6 [(google.api.field_behavior) = OPTIONAL];
  637. }
  638. // A request to instantiate an inline workflow template.
  639. message InstantiateInlineWorkflowTemplateRequest {
  640. // Required. The resource name of the region or location, as described
  641. // in https://cloud.google.com/apis/design/resource_names.
  642. //
  643. // * For `projects.regions.workflowTemplates,instantiateinline`, the resource
  644. // name of the region has the following format:
  645. // `projects/{project_id}/regions/{region}`
  646. //
  647. // * For `projects.locations.workflowTemplates.instantiateinline`, the
  648. // resource name of the location has the following format:
  649. // `projects/{project_id}/locations/{location}`
  650. string parent = 1 [
  651. (google.api.field_behavior) = REQUIRED,
  652. (google.api.resource_reference) = {
  653. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  654. }
  655. ];
  656. // Required. The workflow template to instantiate.
  657. WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
  658. // Optional. A tag that prevents multiple concurrent workflow
  659. // instances with the same tag from running. This mitigates risk of
  660. // concurrent instances started due to retries.
  661. //
  662. // It is recommended to always set this value to a
  663. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  664. //
  665. // The tag must contain only letters (a-z, A-Z), numbers (0-9),
  666. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  667. string request_id = 3 [(google.api.field_behavior) = OPTIONAL];
  668. }
  669. // A request to update a workflow template.
  670. message UpdateWorkflowTemplateRequest {
  671. // Required. The updated workflow template.
  672. //
  673. // The `template.version` field must match the current version.
  674. WorkflowTemplate template = 1 [(google.api.field_behavior) = REQUIRED];
  675. }
  676. // A request to list workflow templates in a project.
  677. message ListWorkflowTemplatesRequest {
  678. // Required. The resource name of the region or location, as described
  679. // in https://cloud.google.com/apis/design/resource_names.
  680. //
  681. // * For `projects.regions.workflowTemplates,list`, the resource
  682. // name of the region has the following format:
  683. // `projects/{project_id}/regions/{region}`
  684. //
  685. // * For `projects.locations.workflowTemplates.list`, the
  686. // resource name of the location has the following format:
  687. // `projects/{project_id}/locations/{location}`
  688. string parent = 1 [
  689. (google.api.field_behavior) = REQUIRED,
  690. (google.api.resource_reference) = {
  691. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  692. }
  693. ];
  694. // Optional. The maximum number of results to return in each response.
  695. int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
  696. // Optional. The page token, returned by a previous call, to request the
  697. // next page of results.
  698. string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
  699. }
  700. // A response to a request to list workflow templates in a project.
  701. message ListWorkflowTemplatesResponse {
  702. // Output only. WorkflowTemplates list.
  703. repeated WorkflowTemplate templates = 1
  704. [(google.api.field_behavior) = OUTPUT_ONLY];
  705. // Output only. This token is included in the response if there are more
  706. // results to fetch. To fetch additional results, provide this value as the
  707. // page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
  708. string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  709. }
  710. // A request to delete a workflow template.
  711. //
  712. // Currently started workflows will remain running.
  713. message DeleteWorkflowTemplateRequest {
  714. // Required. The resource name of the workflow template, as described
  715. // in https://cloud.google.com/apis/design/resource_names.
  716. //
  717. // * For `projects.regions.workflowTemplates.delete`, the resource name
  718. // of the template has the following format:
  719. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  720. //
  721. // * For `projects.locations.workflowTemplates.instantiate`, the resource name
  722. // of the template has the following format:
  723. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  724. string name = 1 [
  725. (google.api.field_behavior) = REQUIRED,
  726. (google.api.resource_reference) = {
  727. type: "dataproc.googleapis.com/WorkflowTemplate"
  728. }
  729. ];
  730. // Optional. The version of workflow template to delete. If specified,
  731. // will only delete the template if the current server version matches
  732. // specified version.
  733. int32 version = 2 [(google.api.field_behavior) = OPTIONAL];
  734. }