workflow_templates.proto 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848
  1. // Copyright 2020 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. syntax = "proto3";
  15. package google.cloud.dataproc.v1beta2;
  16. import "google/api/annotations.proto";
  17. import "google/api/client.proto";
  18. import "google/api/field_behavior.proto";
  19. import "google/api/resource.proto";
  20. import "google/cloud/dataproc/v1beta2/clusters.proto";
  21. import "google/cloud/dataproc/v1beta2/jobs.proto";
  22. import "google/longrunning/operations.proto";
  23. import "google/protobuf/duration.proto";
  24. import "google/protobuf/empty.proto";
  25. import "google/protobuf/timestamp.proto";
  26. option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1beta2;dataproc";
  27. option java_multiple_files = true;
  28. option java_outer_classname = "WorkflowTemplatesProto";
  29. option java_package = "com.google.cloud.dataproc.v1beta2";
  30. // The API interface for managing Workflow Templates in the
  31. // Dataproc API.
  32. service WorkflowTemplateService {
  33. option (google.api.default_host) = "dataproc.googleapis.com";
  34. option (google.api.oauth_scopes) =
  35. "https://www.googleapis.com/auth/cloud-platform";
  36. // Creates new workflow template.
  37. rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest)
  38. returns (WorkflowTemplate) {
  39. option (google.api.http) = {
  40. post: "/v1beta2/{parent=projects/*/regions/*}/workflowTemplates"
  41. body: "template"
  42. additional_bindings {
  43. post: "/v1beta2/{parent=projects/*/locations/*}/workflowTemplates"
  44. body: "template"
  45. }
  46. };
  47. option (google.api.method_signature) = "parent, template";
  48. }
  49. // Retrieves the latest workflow template.
  50. //
  51. // Can retrieve previously instantiated template by specifying optional
  52. // version parameter.
  53. rpc GetWorkflowTemplate(GetWorkflowTemplateRequest)
  54. returns (WorkflowTemplate) {
  55. option (google.api.http) = {
  56. get: "/v1beta2/{name=projects/*/regions/*/workflowTemplates/*}"
  57. additional_bindings {
  58. get: "/v1beta2/{name=projects/*/locations/*/workflowTemplates/*}"
  59. }
  60. };
  61. option (google.api.method_signature) = "name";
  62. }
  63. // Instantiates a template and begins execution.
  64. //
  65. // The returned Operation can be used to track execution of
  66. // workflow by polling
  67. // [operations.get][google.longrunning.Operations.GetOperation].
  68. // The Operation will complete when entire workflow is finished.
  69. //
  70. // The running workflow can be aborted via
  71. // [operations.cancel][google.longrunning.Operations.CancelOperation].
  72. // This will cause any inflight jobs to be cancelled and workflow-owned
  73. // clusters to be deleted.
  74. //
  75. // The [Operation.metadata][google.longrunning.Operation.metadata] will be
  76. // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1beta2#workflowmetadata).
  77. // Also see [Using
  78. // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
  79. //
  80. // On successful completion,
  81. // [Operation.response][google.longrunning.Operation.response] will be
  82. // [Empty][google.protobuf.Empty].
  83. rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest)
  84. returns (google.longrunning.Operation) {
  85. option (google.api.http) = {
  86. post: "/v1beta2/{name=projects/*/regions/*/workflowTemplates/*}:instantiate"
  87. body: "*"
  88. additional_bindings {
  89. post: "/v1beta2/{name=projects/*/locations/*/workflowTemplates/*}:instantiate"
  90. body: "*"
  91. }
  92. };
  93. option (google.api.method_signature) = "name";
  94. option (google.api.method_signature) = "name, parameters";
  95. option (google.longrunning.operation_info) = {
  96. response_type: "google.protobuf.Empty"
  97. metadata_type: "WorkflowMetadata"
  98. };
  99. }
  100. // Instantiates a template and begins execution.
  101. //
  102. // This method is equivalent to executing the sequence
  103. // [CreateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.CreateWorkflowTemplate],
  104. // [InstantiateWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.InstantiateWorkflowTemplate],
  105. // [DeleteWorkflowTemplate][google.cloud.dataproc.v1beta2.WorkflowTemplateService.DeleteWorkflowTemplate].
  106. //
  107. // The returned Operation can be used to track execution of
  108. // workflow by polling
  109. // [operations.get][google.longrunning.Operations.GetOperation].
  110. // The Operation will complete when entire workflow is finished.
  111. //
  112. // The running workflow can be aborted via
  113. // [operations.cancel][google.longrunning.Operations.CancelOperation].
  114. // This will cause any inflight jobs to be cancelled and workflow-owned
  115. // clusters to be deleted.
  116. //
  117. // The [Operation.metadata][google.longrunning.Operation.metadata] will be
  118. // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata).
  119. // Also see [Using
  120. // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata).
  121. //
  122. // On successful completion,
  123. // [Operation.response][google.longrunning.Operation.response] will be
  124. // [Empty][google.protobuf.Empty].
  125. rpc InstantiateInlineWorkflowTemplate(
  126. InstantiateInlineWorkflowTemplateRequest)
  127. returns (google.longrunning.Operation) {
  128. option (google.api.http) = {
  129. post: "/v1beta2/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline"
  130. body: "template"
  131. additional_bindings {
  132. post: "/v1beta2/{parent=projects/*/regions/*}/workflowTemplates:instantiateInline"
  133. body: "template"
  134. }
  135. };
  136. option (google.api.method_signature) = "parent, template";
  137. option (google.longrunning.operation_info) = {
  138. response_type: "google.protobuf.Empty"
  139. metadata_type: "WorkflowMetadata"
  140. };
  141. }
  142. // Updates (replaces) workflow template. The updated template
  143. // must contain version that matches the current server version.
  144. rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest)
  145. returns (WorkflowTemplate) {
  146. option (google.api.http) = {
  147. put: "/v1beta2/{template.name=projects/*/regions/*/workflowTemplates/*}"
  148. body: "template"
  149. additional_bindings {
  150. put: "/v1beta2/{template.name=projects/*/locations/*/workflowTemplates/*}"
  151. body: "template"
  152. }
  153. };
  154. option (google.api.method_signature) = "template";
  155. }
  156. // Lists workflows that match the specified filter in the request.
  157. rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest)
  158. returns (ListWorkflowTemplatesResponse) {
  159. option (google.api.http) = {
  160. get: "/v1beta2/{parent=projects/*/regions/*}/workflowTemplates"
  161. additional_bindings {
  162. get: "/v1beta2/{parent=projects/*/locations/*}/workflowTemplates"
  163. }
  164. };
  165. option (google.api.method_signature) = "parent";
  166. }
  167. // Deletes a workflow template. It does not cancel in-progress workflows.
  168. rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest)
  169. returns (google.protobuf.Empty) {
  170. option (google.api.http) = {
  171. delete: "/v1beta2/{name=projects/*/regions/*/workflowTemplates/*}"
  172. additional_bindings {
  173. delete: "/v1beta2/{name=projects/*/locations/*/workflowTemplates/*}"
  174. }
  175. };
  176. option (google.api.method_signature) = "name";
  177. }
  178. }
  179. // A Dataproc workflow template resource.
  180. message WorkflowTemplate {
  181. option (google.api.resource) = {
  182. type: "dataproc.googleapis.com/WorkflowTemplate"
  183. pattern: "projects/{project}/regions/{region}/workflowTemplates/{workflow_template}"
  184. pattern: "projects/{project}/locations/{location}/workflowTemplates/{workflow_template}"
  185. history: ORIGINALLY_SINGLE_PATTERN
  186. };
  187. // Required. The template id.
  188. //
  189. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  190. // underscores (_), and hyphens (-). Cannot begin or end with underscore
  191. // or hyphen. Must consist of between 3 and 50 characters.
  192. //
  193. // .
  194. string id = 2 [(google.api.field_behavior) = REQUIRED];
  195. // Output only. The resource name of the workflow template, as described
  196. // in https://cloud.google.com/apis/design/resource_names.
  197. //
  198. // * For `projects.regions.workflowTemplates`, the resource name of the
  199. // template has the following format:
  200. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  201. //
  202. // * For `projects.locations.workflowTemplates`, the resource name of the
  203. // template has the following format:
  204. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  205. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  206. // Optional. Used to perform a consistent read-modify-write.
  207. //
  208. // This field should be left blank for a `CreateWorkflowTemplate` request. It
  209. // is required for an `UpdateWorkflowTemplate` request, and must match the
  210. // current server version. A typical update template flow would fetch the
  211. // current template with a `GetWorkflowTemplate` request, which will return
  212. // the current template with the `version` field filled in with the
  213. // current server version. The user updates other fields in the template,
  214. // then returns it as part of the `UpdateWorkflowTemplate` request.
  215. int32 version = 3 [(google.api.field_behavior) = OPTIONAL];
  216. // Output only. The time template was created.
  217. google.protobuf.Timestamp create_time = 4
  218. [(google.api.field_behavior) = OUTPUT_ONLY];
  219. // Output only. The time template was last updated.
  220. google.protobuf.Timestamp update_time = 5
  221. [(google.api.field_behavior) = OUTPUT_ONLY];
  222. // Optional. The labels to associate with this template. These labels
  223. // will be propagated to all jobs and clusters created by the workflow
  224. // instance.
  225. //
  226. // Label **keys** must contain 1 to 63 characters, and must conform to
  227. // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
  228. //
  229. // Label **values** may be empty, but, if present, must contain 1 to 63
  230. // characters, and must conform to
  231. // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
  232. //
  233. // No more than 32 labels can be associated with a template.
  234. map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
  235. // Required. WorkflowTemplate scheduling information.
  236. WorkflowTemplatePlacement placement = 7;
  237. // Required. The Directed Acyclic Graph of Jobs to submit.
  238. repeated OrderedJob jobs = 8;
  239. // Optional. Template parameters whose values are substituted into the
  240. // template. Values for parameters must be provided when the template is
  241. // instantiated.
  242. repeated TemplateParameter parameters = 9
  243. [(google.api.field_behavior) = OPTIONAL];
  244. // Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h",
  245. // and "d" suffixes for second, minute, hour, and day duration values,
  246. // respectively. The timeout duration must be from 10 minutes ("10m") to 24
  247. // hours ("24h" or "1d"). The timer begins when the first job is submitted. If
  248. // the workflow is running at the end of the timeout period, any remaining
  249. // jobs are cancelled, the workflow is terminated, and if the workflow was
  250. // running on a [managed
  251. // cluster](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster),
  252. // the cluster is deleted.
  253. google.protobuf.Duration dag_timeout = 10
  254. [(google.api.field_behavior) = OPTIONAL];
  255. }
  256. // Specifies workflow execution target.
  257. //
  258. // Either `managed_cluster` or `cluster_selector` is required.
  259. message WorkflowTemplatePlacement {
  260. // Required. Specifies where workflow executes; either on a managed
  261. // cluster or an existing cluster chosen by labels.
  262. oneof placement {
  263. // Optional. A cluster that is managed by the workflow.
  264. ManagedCluster managed_cluster = 1;
  265. // Optional. A selector that chooses target cluster for jobs based
  266. // on metadata.
  267. //
  268. // The selector is evaluated at the time each job is submitted.
  269. ClusterSelector cluster_selector = 2;
  270. }
  271. }
  272. // Cluster that is managed by the workflow.
  273. message ManagedCluster {
  274. // Required. The cluster name prefix. A unique cluster name will be formed by
  275. // appending a random suffix.
  276. //
  277. // The name must contain only lower-case letters (a-z), numbers (0-9),
  278. // and hyphens (-). Must begin with a letter. Cannot begin or end with
  279. // hyphen. Must consist of between 2 and 35 characters.
  280. string cluster_name = 2;
  281. // Required. The cluster configuration.
  282. ClusterConfig config = 3;
  283. // Optional. The labels to associate with this cluster.
  284. //
  285. // Label keys must be between 1 and 63 characters long, and must conform to
  286. // the following PCRE regular expression:
  287. // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
  288. //
  289. // Label values must be between 1 and 63 characters long, and must conform to
  290. // the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
  291. //
  292. // No more than 32 labels can be associated with a given cluster.
  293. map<string, string> labels = 4;
  294. }
  295. // A selector that chooses target cluster for jobs based on metadata.
  296. message ClusterSelector {
  297. // Optional. The zone where workflow process executes. This parameter does not
  298. // affect the selection of the cluster.
  299. //
  300. // If unspecified, the zone of the first cluster matching the selector
  301. // is used.
  302. string zone = 1;
  303. // Required. The cluster labels. Cluster must have all labels
  304. // to match.
  305. map<string, string> cluster_labels = 2;
  306. }
  307. // A job executed by the workflow.
  308. message OrderedJob {
  309. // Required. The step id. The id must be unique among all jobs
  310. // within the template.
  311. //
  312. // The step id is used as prefix for job id, as job
  313. // `goog-dataproc-workflow-step-id` label, and in
  314. // [prerequisiteStepIds][google.cloud.dataproc.v1beta2.OrderedJob.prerequisite_step_ids]
  315. // field from other steps.
  316. //
  317. // The id must contain only letters (a-z, A-Z), numbers (0-9),
  318. // underscores (_), and hyphens (-). Cannot begin or end with underscore
  319. // or hyphen. Must consist of between 3 and 50 characters.
  320. string step_id = 1 [(google.api.field_behavior) = REQUIRED];
  321. // Required. The job definition.
  322. oneof job_type {
  323. // Optional. Job is a Hadoop job.
  324. HadoopJob hadoop_job = 2 [(google.api.field_behavior) = OPTIONAL];
  325. // Optional. Job is a Spark job.
  326. SparkJob spark_job = 3 [(google.api.field_behavior) = OPTIONAL];
  327. // Optional. Job is a PySpark job.
  328. PySparkJob pyspark_job = 4 [(google.api.field_behavior) = OPTIONAL];
  329. // Optional. Job is a Hive job.
  330. HiveJob hive_job = 5 [(google.api.field_behavior) = OPTIONAL];
  331. // Optional. Job is a Pig job.
  332. PigJob pig_job = 6 [(google.api.field_behavior) = OPTIONAL];
  333. // Optional. Job is a SparkR job.
  334. SparkRJob spark_r_job = 11 [(google.api.field_behavior) = OPTIONAL];
  335. // Optional. Job is a SparkSql job.
  336. SparkSqlJob spark_sql_job = 7 [(google.api.field_behavior) = OPTIONAL];
  337. // Optional. Job is a Presto job.
  338. PrestoJob presto_job = 12 [(google.api.field_behavior) = OPTIONAL];
  339. }
  340. // Optional. The labels to associate with this job.
  341. //
  342. // Label keys must be between 1 and 63 characters long, and must conform to
  343. // the following regular expression:
  344. // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
  345. //
  346. // Label values must be between 1 and 63 characters long, and must conform to
  347. // the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
  348. //
  349. // No more than 32 labels can be associated with a given job.
  350. map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL];
  351. // Optional. Job scheduling configuration.
  352. JobScheduling scheduling = 9 [(google.api.field_behavior) = OPTIONAL];
  353. // Optional. The optional list of prerequisite job step_ids.
  354. // If not specified, the job will start at the beginning of workflow.
  355. repeated string prerequisite_step_ids = 10
  356. [(google.api.field_behavior) = OPTIONAL];
  357. }
  358. // A configurable parameter that replaces one or more fields in the template.
  359. // Parameterizable fields:
  360. // - Labels
  361. // - File uris
  362. // - Job properties
  363. // - Job arguments
  364. // - Script variables
  365. // - Main class (in HadoopJob and SparkJob)
  366. // - Zone (in ClusterSelector)
  367. message TemplateParameter {
  368. // Required. Parameter name.
  369. // The parameter name is used as the key, and paired with the
  370. // parameter value, which are passed to the template when the template
  371. // is instantiated.
  372. // The name must contain only capital letters (A-Z), numbers (0-9), and
  373. // underscores (_), and must not start with a number. The maximum length is
  374. // 40 characters.
  375. string name = 1;
  376. // Required. Paths to all fields that the parameter replaces.
  377. // A field is allowed to appear in at most one parameter's list of field
  378. // paths.
  379. //
  380. // A field path is similar in syntax to a
  381. // [google.protobuf.FieldMask][google.protobuf.FieldMask]. For example, a
  382. // field path that references the zone field of a workflow template's cluster
  383. // selector would be specified as `placement.clusterSelector.zone`.
  384. //
  385. // Also, field paths can reference fields using the following syntax:
  386. //
  387. // * Values in maps can be referenced by key:
  388. // * labels['key']
  389. // * placement.clusterSelector.clusterLabels['key']
  390. // * placement.managedCluster.labels['key']
  391. // * placement.clusterSelector.clusterLabels['key']
  392. // * jobs['step-id'].labels['key']
  393. //
  394. // * Jobs in the jobs list can be referenced by step-id:
  395. // * jobs['step-id'].hadoopJob.mainJarFileUri
  396. // * jobs['step-id'].hiveJob.queryFileUri
  397. // * jobs['step-id'].pySparkJob.mainPythonFileUri
  398. // * jobs['step-id'].hadoopJob.jarFileUris[0]
  399. // * jobs['step-id'].hadoopJob.archiveUris[0]
  400. // * jobs['step-id'].hadoopJob.fileUris[0]
  401. // * jobs['step-id'].pySparkJob.pythonFileUris[0]
  402. //
  403. // * Items in repeated fields can be referenced by a zero-based index:
  404. // * jobs['step-id'].sparkJob.args[0]
  405. //
  406. // * Other examples:
  407. // * jobs['step-id'].hadoopJob.properties['key']
  408. // * jobs['step-id'].hadoopJob.args[0]
  409. // * jobs['step-id'].hiveJob.scriptVariables['key']
  410. // * jobs['step-id'].hadoopJob.mainJarFileUri
  411. // * placement.clusterSelector.zone
  412. //
  413. // It may not be possible to parameterize maps and repeated fields in their
  414. // entirety since only individual map values and individual items in repeated
  415. // fields can be referenced. For example, the following field paths are
  416. // invalid:
  417. //
  418. // - placement.clusterSelector.clusterLabels
  419. // - jobs['step-id'].sparkJob.args
  420. repeated string fields = 2;
  421. // Optional. Brief description of the parameter.
  422. // Must not exceed 1024 characters.
  423. string description = 3;
  424. // Optional. Validation rules to be applied to this parameter's value.
  425. ParameterValidation validation = 4;
  426. }
  427. // Configuration for parameter validation.
  428. message ParameterValidation {
  429. // Required. The type of validation to be performed.
  430. oneof validation_type {
  431. // Validation based on regular expressions.
  432. RegexValidation regex = 1;
  433. // Validation based on a list of allowed values.
  434. ValueValidation values = 2;
  435. }
  436. }
  437. // Validation based on regular expressions.
  438. message RegexValidation {
  439. // Required. RE2 regular expressions used to validate the parameter's value.
  440. // The value must match the regex in its entirety (substring
  441. // matches are not sufficient).
  442. repeated string regexes = 1;
  443. }
  444. // Validation based on a list of allowed values.
  445. message ValueValidation {
  446. // Required. List of allowed values for the parameter.
  447. repeated string values = 1;
  448. }
  449. // A Dataproc workflow template resource.
  450. message WorkflowMetadata {
  451. // The operation state.
  452. enum State {
  453. // Unused.
  454. UNKNOWN = 0;
  455. // The operation has been created.
  456. PENDING = 1;
  457. // The operation is running.
  458. RUNNING = 2;
  459. // The operation is done; either cancelled or completed.
  460. DONE = 3;
  461. }
  462. // Output only. The resource name of the workflow template as described
  463. // in https://cloud.google.com/apis/design/resource_names.
  464. //
  465. // * For `projects.regions.workflowTemplates`, the resource name of the
  466. // template has the following format:
  467. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  468. //
  469. // * For `projects.locations.workflowTemplates`, the resource name of the
  470. // template has the following format:
  471. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  472. string template = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  473. // Output only. The version of template at the time of
  474. // workflow instantiation.
  475. int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  476. // Output only. The create cluster operation metadata.
  477. ClusterOperation create_cluster = 3
  478. [(google.api.field_behavior) = OUTPUT_ONLY];
  479. // Output only. The workflow graph.
  480. WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  481. // Output only. The delete cluster operation metadata.
  482. ClusterOperation delete_cluster = 5
  483. [(google.api.field_behavior) = OUTPUT_ONLY];
  484. // Output only. The workflow state.
  485. State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  486. // Output only. The name of the target cluster.
  487. string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
  488. // Map from parameter names to values that were used for those parameters.
  489. map<string, string> parameters = 8;
  490. // Output only. Workflow start time.
  491. google.protobuf.Timestamp start_time = 9
  492. [(google.api.field_behavior) = OUTPUT_ONLY];
  493. // Output only. Workflow end time.
  494. google.protobuf.Timestamp end_time = 10
  495. [(google.api.field_behavior) = OUTPUT_ONLY];
  496. // Output only. The UUID of target cluster.
  497. string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
  498. // Output only. The timeout duration for the DAG of jobs.
  499. // Minimum timeout duration is 10 minutes and maximum is 24 hours, expressed
  500. // as a
  501. // [google.protobuf.Duration][https://developers.google.com/protocol-buffers/docs/proto3#json_mapping].
  502. // For example, "1800" = 1800 seconds/30 minutes duration.
  503. google.protobuf.Duration dag_timeout = 12
  504. [(google.api.field_behavior) = OUTPUT_ONLY];
  505. // Output only. DAG start time, which is only set for workflows with
  506. // [dag_timeout][google.cloud.dataproc.v1beta2.WorkflowMetadata.dag_timeout]
  507. // when the DAG begins.
  508. google.protobuf.Timestamp dag_start_time = 13
  509. [(google.api.field_behavior) = OUTPUT_ONLY];
  510. // Output only. DAG end time, which is only set for workflows with
  511. // [dag_timeout][google.cloud.dataproc.v1beta2.WorkflowMetadata.dag_timeout]
  512. // when the DAG ends.
  513. google.protobuf.Timestamp dag_end_time = 14
  514. [(google.api.field_behavior) = OUTPUT_ONLY];
  515. }
  516. // The cluster operation triggered by a workflow.
  517. message ClusterOperation {
  518. // Output only. The id of the cluster operation.
  519. string operation_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  520. // Output only. Error, if operation failed.
  521. string error = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  522. // Output only. Indicates the operation is done.
  523. bool done = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  524. }
  525. // The workflow graph.
  526. message WorkflowGraph {
  527. // Output only. The workflow nodes.
  528. repeated WorkflowNode nodes = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  529. }
  530. // The workflow node.
  531. message WorkflowNode {
  532. // The workflow node state.
  533. enum NodeState {
  534. // State is unspecified.
  535. NODE_STATUS_UNSPECIFIED = 0;
  536. // The node is awaiting prerequisite node to finish.
  537. BLOCKED = 1;
  538. // The node is runnable but not running.
  539. RUNNABLE = 2;
  540. // The node is running.
  541. RUNNING = 3;
  542. // The node completed successfully.
  543. COMPLETED = 4;
  544. // The node failed. A node can be marked FAILED because
  545. // its ancestor or peer failed.
  546. FAILED = 5;
  547. }
  548. // Output only. The name of the node.
  549. string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  550. // Output only. Node's prerequisite nodes.
  551. repeated string prerequisite_step_ids = 2
  552. [(google.api.field_behavior) = OUTPUT_ONLY];
  553. // Output only. The job id; populated after the node enters RUNNING state.
  554. string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  555. // Output only. The node state.
  556. NodeState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  557. // Output only. The error detail.
  558. string error = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
  559. }
  560. // A request to create a workflow template.
  561. message CreateWorkflowTemplateRequest {
  562. // Required. The resource name of the region or location, as described
  563. // in https://cloud.google.com/apis/design/resource_names.
  564. //
  565. // * For `projects.regions.workflowTemplates,create`, the resource name of the
  566. // region has the following format:
  567. // `projects/{project_id}/regions/{region}`
  568. //
  569. // * For `projects.locations.workflowTemplates.create`, the resource name of
  570. // the location has the following format:
  571. // `projects/{project_id}/locations/{location}`
  572. string parent = 1 [
  573. (google.api.field_behavior) = REQUIRED,
  574. (google.api.resource_reference) = {
  575. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  576. }
  577. ];
  578. // Required. The Dataproc workflow template to create.
  579. WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
  580. }
  581. // A request to fetch a workflow template.
  582. message GetWorkflowTemplateRequest {
  583. // Required. The resource name of the workflow template, as described
  584. // in https://cloud.google.com/apis/design/resource_names.
  585. //
  586. // * For `projects.regions.workflowTemplates.get`, the resource name of the
  587. // template has the following format:
  588. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  589. //
  590. // * For `projects.locations.workflowTemplates.get`, the resource name of the
  591. // template has the following format:
  592. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  593. string name = 1 [
  594. (google.api.field_behavior) = REQUIRED,
  595. (google.api.resource_reference) = {
  596. type: "dataproc.googleapis.com/WorkflowTemplate"
  597. }
  598. ];
  599. // Optional. The version of workflow template to retrieve. Only previously
  600. // instantiated versions can be retrieved.
  601. //
  602. // If unspecified, retrieves the current version.
  603. int32 version = 2;
  604. }
  605. // A request to instantiate a workflow template.
  606. message InstantiateWorkflowTemplateRequest {
  607. // Required. The resource name of the workflow template, as described
  608. // in https://cloud.google.com/apis/design/resource_names.
  609. //
  610. // * For `projects.regions.workflowTemplates.instantiate`, the resource name
  611. // of the template has the following format:
  612. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  613. //
  614. // * For `projects.locations.workflowTemplates.instantiate`, the resource name
  615. // of the template has the following format:
  616. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  617. string name = 1 [
  618. (google.api.field_behavior) = REQUIRED,
  619. (google.api.resource_reference) = {
  620. type: "dataproc.googleapis.com/WorkflowTemplate"
  621. }
  622. ];
  623. // Optional. The version of workflow template to instantiate. If specified,
  624. // the workflow will be instantiated only if the current version of
  625. // the workflow template has the supplied version.
  626. //
  627. // This option cannot be used to instantiate a previous version of
  628. // workflow template.
  629. int32 version = 2;
  630. // Deprecated. Please use `request_id` field instead.
  631. string instance_id = 3 [deprecated = true];
  632. // Optional. A tag that prevents multiple concurrent workflow
  633. // instances with the same tag from running. This mitigates risk of
  634. // concurrent instances started due to retries.
  635. //
  636. // It is recommended to always set this value to a
  637. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  638. //
  639. // The tag must contain only letters (a-z, A-Z), numbers (0-9),
  640. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  641. string request_id = 5;
  642. // Optional. Map from parameter names to values that should be used for those
  643. // parameters. Values may not exceed 100 characters.
  644. map<string, string> parameters = 4;
  645. }
  646. // A request to instantiate an inline workflow template.
  647. message InstantiateInlineWorkflowTemplateRequest {
  648. // Required. The resource name of the region or location, as described
  649. // in https://cloud.google.com/apis/design/resource_names.
  650. //
  651. // * For `projects.regions.workflowTemplates,instantiateinline`, the resource
  652. // name of the region has the following format:
  653. // `projects/{project_id}/regions/{region}`
  654. //
  655. // * For `projects.locations.workflowTemplates.instantiateinline`, the
  656. // resource name of the location has the following format:
  657. // `projects/{project_id}/locations/{location}`
  658. string parent = 1 [
  659. (google.api.field_behavior) = REQUIRED,
  660. (google.api.resource_reference) = {
  661. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  662. }
  663. ];
  664. // Required. The workflow template to instantiate.
  665. WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED];
  666. // Deprecated. Please use `request_id` field instead.
  667. string instance_id = 3;
  668. // Optional. A tag that prevents multiple concurrent workflow
  669. // instances with the same tag from running. This mitigates risk of
  670. // concurrent instances started due to retries.
  671. //
  672. // It is recommended to always set this value to a
  673. // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
  674. //
  675. // The tag must contain only letters (a-z, A-Z), numbers (0-9),
  676. // underscores (_), and hyphens (-). The maximum length is 40 characters.
  677. string request_id = 4;
  678. }
  679. // A request to update a workflow template.
  680. message UpdateWorkflowTemplateRequest {
  681. // Required. The updated workflow template.
  682. //
  683. // The `template.version` field must match the current version.
  684. WorkflowTemplate template = 1 [(google.api.field_behavior) = REQUIRED];
  685. }
  686. // A request to list workflow templates in a project.
  687. message ListWorkflowTemplatesRequest {
  688. // Required. The resource name of the region or location, as described
  689. // in https://cloud.google.com/apis/design/resource_names.
  690. //
  691. // * For `projects.regions.workflowTemplates,list`, the resource
  692. // name of the region has the following format:
  693. // `projects/{project_id}/regions/{region}`
  694. //
  695. // * For `projects.locations.workflowTemplates.list`, the
  696. // resource name of the location has the following format:
  697. // `projects/{project_id}/locations/{location}`
  698. string parent = 1 [
  699. (google.api.field_behavior) = REQUIRED,
  700. (google.api.resource_reference) = {
  701. child_type: "dataproc.googleapis.com/WorkflowTemplate"
  702. }
  703. ];
  704. // Optional. The maximum number of results to return in each response.
  705. int32 page_size = 2;
  706. // Optional. The page token, returned by a previous call, to request the
  707. // next page of results.
  708. string page_token = 3;
  709. }
  710. // A response to a request to list workflow templates in a project.
  711. message ListWorkflowTemplatesResponse {
  712. // Output only. WorkflowTemplates list.
  713. repeated WorkflowTemplate templates = 1
  714. [(google.api.field_behavior) = OUTPUT_ONLY];
  715. // Output only. This token is included in the response if there are more
  716. // results to fetch. To fetch additional results, provide this value as the
  717. // page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
  718. string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
  719. }
  720. // A request to delete a workflow template.
  721. //
  722. // Currently started workflows will remain running.
  723. message DeleteWorkflowTemplateRequest {
  724. // Required. The resource name of the workflow template, as described
  725. // in https://cloud.google.com/apis/design/resource_names.
  726. //
  727. // * For `projects.regions.workflowTemplates.delete`, the resource name
  728. // of the template has the following format:
  729. // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
  730. //
  731. // * For `projects.locations.workflowTemplates.instantiate`, the resource name
  732. // of the template has the following format:
  733. // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}`
  734. string name = 1 [
  735. (google.api.field_behavior) = REQUIRED,
  736. (google.api.resource_reference) = {
  737. type: "dataproc.googleapis.com/WorkflowTemplate"
  738. }
  739. ];
  740. // Optional. The version of workflow template to delete. If specified,
  741. // will only delete the template if the current server version matches
  742. // specified version.
  743. int32 version = 2;
  744. }