123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613 |
- // Copyright 2017 Google Inc.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- syntax = "proto3";
- package google.cloud.ml.v1;
- import "google/api/annotations.proto";
- import "google/api/auth.proto";
- import "google/protobuf/empty.proto";
- import "google/protobuf/timestamp.proto";
- option go_package = "google.golang.org/genproto/googleapis/cloud/ml/v1;ml";
- option java_multiple_files = true;
- option java_outer_classname = "JobServiceProto";
- option java_package = "com.google.cloud.ml.api.v1";
- // Copyright 2017 Google Inc. All Rights Reserved.
- //
- // Proto file for the Google Cloud Machine Learning Engine.
- // Describes the 'job service' to manage training and prediction jobs.
- // Service to create and manage training and batch prediction jobs.
- service JobService {
- // Creates a training or a batch prediction job.
- rpc CreateJob(CreateJobRequest) returns (Job) {
- option (google.api.http) = {
- post: "/v1/{parent=projects/*}/jobs"
- body: "job"
- };
- }
- // Lists the jobs in the project.
- rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) {
- option (google.api.http) = {
- get: "/v1/{parent=projects/*}/jobs"
- };
- }
- // Describes a job.
- rpc GetJob(GetJobRequest) returns (Job) {
- option (google.api.http) = {
- get: "/v1/{name=projects/*/jobs/*}"
- };
- }
- // Cancels a running job.
- rpc CancelJob(CancelJobRequest) returns (google.protobuf.Empty) {
- option (google.api.http) = {
- post: "/v1/{name=projects/*/jobs/*}:cancel"
- body: "*"
- };
- }
- }
- // Represents input parameters for a training job.
- message TrainingInput {
- // A scale tier is an abstract representation of the resources Cloud ML
- // will allocate to a training job. When selecting a scale tier for your
- // training job, you should consider the size of your training dataset and
- // the complexity of your model. As the tiers increase, virtual machines are
- // added to handle your job, and the individual machines in the cluster
- // generally have more memory and greater processing power than they do at
- // lower tiers. The number of training units charged per hour of processing
- // increases as tiers get more advanced. Refer to the
- // [pricing guide](/ml/pricing) for more details. Note that in addition to
- // incurring costs, your use of training resources is constrained by the
- // [quota policy](/ml/quota).
- enum ScaleTier {
- // A single worker instance. This tier is suitable for learning how to use
- // Cloud ML, and for experimenting with new models using small datasets.
- BASIC = 0;
- // Many workers and a few parameter servers.
- STANDARD_1 = 1;
- // A large number of workers with many parameter servers.
- PREMIUM_1 = 3;
- // A single worker instance [with a GPU](ml/docs/how-tos/using-gpus).
- BASIC_GPU = 6;
- // The CUSTOM tier is not a set tier, but rather enables you to use your
- // own cluster specification. When you use this tier, set values to
- // configure your processing cluster according to these guidelines:
- //
- // * You _must_ set `TrainingInput.masterType` to specify the type
- // of machine to use for your master node. This is the only required
- // setting.
- //
- // * You _may_ set `TrainingInput.workerCount` to specify the number of
- // workers to use. If you specify one or more workers, you _must_ also
- // set `TrainingInput.workerType` to specify the type of machine to use
- // for your worker nodes.
- //
- // * You _may_ set `TrainingInput.parameterServerCount` to specify the
- // number of parameter servers to use. If you specify one or more
- // parameter servers, you _must_ also set
- // `TrainingInput.parameterServerType` to specify the type of machine to
- // use for your parameter servers.
- //
- // Note that all of your workers must use the same machine type, which can
- // be different from your parameter server type and master type. Your
- // parameter servers must likewise use the same machine type, which can be
- // different from your worker type and master type.
- CUSTOM = 5;
- }
- // Required. Specifies the machine types, the number of replicas for workers
- // and parameter servers.
- ScaleTier scale_tier = 1;
- // Optional. Specifies the type of virtual machine to use for your training
- // job's master worker.
- //
- // The following types are supported:
- //
- // <dl>
- // <dt>standard</dt>
- // <dd>
- // A basic machine configuration suitable for training simple models with
- // small to moderate datasets.
- // </dd>
- // <dt>large_model</dt>
- // <dd>
- // A machine with a lot of memory, specially suited for parameter servers
- // when your model is large (having many hidden layers or layers with very
- // large numbers of nodes).
- // </dd>
- // <dt>complex_model_s</dt>
- // <dd>
- // A machine suitable for the master and workers of the cluster when your
- // model requires more computation than the standard machine can handle
- // satisfactorily.
- // </dd>
- // <dt>complex_model_m</dt>
- // <dd>
- // A machine with roughly twice the number of cores and roughly double the
- // memory of <code suppresswarning="true">complex_model_s</code>.
- // </dd>
- // <dt>complex_model_l</dt>
- // <dd>
- // A machine with roughly twice the number of cores and roughly double the
- // memory of <code suppresswarning="true">complex_model_m</code>.
- // </dd>
- // <dt>standard_gpu</dt>
- // <dd>
- // A machine equivalent to <code suppresswarning="true">standard</code> that
- // also includes a
- // <a href="ml/docs/how-tos/using-gpus">
- // GPU that you can use in your trainer</a>.
- // </dd>
- // <dt>complex_model_m_gpu</dt>
- // <dd>
- // A machine equivalent to
- // <code suppresswarning="true">coplex_model_m</code> that also includes
- // four GPUs.
- // </dd>
- // </dl>
- //
- // You must set this value when `scaleTier` is set to `CUSTOM`.
- string master_type = 2;
- // Optional. Specifies the type of virtual machine to use for your training
- // job's worker nodes.
- //
- // The supported values are the same as those described in the entry for
- // `masterType`.
- //
- // This value must be present when `scaleTier` is set to `CUSTOM` and
- // `workerCount` is greater than zero.
- string worker_type = 3;
- // Optional. Specifies the type of virtual machine to use for your training
- // job's parameter server.
- //
- // The supported values are the same as those described in the entry for
- // `master_type`.
- //
- // This value must be present when `scaleTier` is set to `CUSTOM` and
- // `parameter_server_count` is greater than zero.
- string parameter_server_type = 4;
- // Optional. The number of worker replicas to use for the training job. Each
- // replica in the cluster will be of the type specified in `worker_type`.
- //
- // This value can only be used when `scale_tier` is set to `CUSTOM`. If you
- // set this value, you must also set `worker_type`.
- int64 worker_count = 5;
- // Optional. The number of parameter server replicas to use for the training
- // job. Each replica in the cluster will be of the type specified in
- // `parameter_server_type`.
- //
- // This value can only be used when `scale_tier` is set to `CUSTOM`.If you
- // set this value, you must also set `parameter_server_type`.
- int64 parameter_server_count = 6;
- // Required. The Google Cloud Storage location of the packages with
- // the training program and any additional dependencies.
- repeated string package_uris = 7;
- // Required. The Python module name to run after installing the packages.
- string python_module = 8;
- // Optional. Command line arguments to pass to the program.
- repeated string args = 10;
- // Optional. The set of Hyperparameters to tune.
- HyperparameterSpec hyperparameters = 12;
- // Required. The Google Compute Engine region to run the training job in.
- string region = 14;
- // Optional. A Google Cloud Storage path in which to store training outputs
- // and other data needed for training. This path is passed to your TensorFlow
- // program as the 'job_dir' command-line argument. The benefit of specifying
- // this field is that Cloud ML validates the path for use in training.
- string job_dir = 16;
- // Optional. The Google Cloud ML runtime version to use for training. If not
- // set, Google Cloud ML will choose the latest stable version.
- string runtime_version = 15;
- }
- // Represents a set of hyperparameters to optimize.
- message HyperparameterSpec {
- // The available types of optimization goals.
- enum GoalType {
- // Goal Type will default to maximize.
- GOAL_TYPE_UNSPECIFIED = 0;
- // Maximize the goal metric.
- MAXIMIZE = 1;
- // Minimize the goal metric.
- MINIMIZE = 2;
- }
- // Required. The type of goal to use for tuning. Available types are
- // `MAXIMIZE` and `MINIMIZE`.
- //
- // Defaults to `MAXIMIZE`.
- GoalType goal = 1;
- // Required. The set of parameters to tune.
- repeated ParameterSpec params = 2;
- // Optional. How many training trials should be attempted to optimize
- // the specified hyperparameters.
- //
- // Defaults to one.
- int32 max_trials = 3;
- // Optional. The number of training trials to run concurrently.
- // You can reduce the time it takes to perform hyperparameter tuning by adding
- // trials in parallel. However, each trail only benefits from the information
- // gained in completed trials. That means that a trial does not get access to
- // the results of trials running at the same time, which could reduce the
- // quality of the overall optimization.
- //
- // Each trial will use the same scale tier and machine types.
- //
- // Defaults to one.
- int32 max_parallel_trials = 4;
- // Optional. The Tensorflow summary tag name to use for optimizing trials. For
- // current versions of Tensorflow, this tag name should exactly match what is
- // shown in Tensorboard, including all scopes. For versions of Tensorflow
- // prior to 0.12, this should be only the tag passed to tf.Summary.
- // By default, "training/hptuning/metric" will be used.
- string hyperparameter_metric_tag = 5;
- }
- // Represents a single hyperparameter to optimize.
- message ParameterSpec {
- // The type of the parameter.
- enum ParameterType {
- // You must specify a valid type. Using this unspecified type will result in
- // an error.
- PARAMETER_TYPE_UNSPECIFIED = 0;
- // Type for real-valued parameters.
- DOUBLE = 1;
- // Type for integral parameters.
- INTEGER = 2;
- // The parameter is categorical, with a value chosen from the categories
- // field.
- CATEGORICAL = 3;
- // The parameter is real valued, with a fixed set of feasible points. If
- // `type==DISCRETE`, feasible_points must be provided, and
- // {`min_value`, `max_value`} will be ignored.
- DISCRETE = 4;
- }
- // The type of scaling that should be applied to this parameter.
- enum ScaleType {
- // By default, no scaling is applied.
- NONE = 0;
- // Scales the feasible space to (0, 1) linearly.
- UNIT_LINEAR_SCALE = 1;
- // Scales the feasible space logarithmically to (0, 1). The entire feasible
- // space must be strictly positive.
- UNIT_LOG_SCALE = 2;
- // Scales the feasible space "reverse" logarithmically to (0, 1). The result
- // is that values close to the top of the feasible space are spread out more
- // than points near the bottom. The entire feasible space must be strictly
- // positive.
- UNIT_REVERSE_LOG_SCALE = 3;
- }
- // Required. The parameter name must be unique amongst all ParameterConfigs in
- // a HyperparameterSpec message. E.g., "learning_rate".
- string parameter_name = 1;
- // Required. The type of the parameter.
- ParameterType type = 4;
- // Required if type is `DOUBLE` or `INTEGER`. This field
- // should be unset if type is `CATEGORICAL`. This value should be integers if
- // type is INTEGER.
- double min_value = 2;
- // Required if typeis `DOUBLE` or `INTEGER`. This field
- // should be unset if type is `CATEGORICAL`. This value should be integers if
- // type is `INTEGER`.
- double max_value = 3;
- // Required if type is `CATEGORICAL`. The list of possible categories.
- repeated string categorical_values = 5;
- // Required if type is `DISCRETE`.
- // A list of feasible points.
- // The list should be in strictly increasing order. For instance, this
- // parameter might have possible settings of 1.5, 2.5, and 4.0. This list
- // should not contain more than 1,000 values.
- repeated double discrete_values = 6;
- // Optional. How the parameter should be scaled to the hypercube.
- // Leave unset for categorical parameters.
- // Some kind of scaling is strongly recommended for real or integral
- // parameters (e.g., `UNIT_LINEAR_SCALE`).
- ScaleType scale_type = 7;
- }
- // Represents the result of a single hyperparameter tuning trial from a
- // training job. The TrainingOutput object that is returned on successful
- // completion of a training job with hyperparameter tuning includes a list
- // of HyperparameterOutput objects, one for each successful trial.
- message HyperparameterOutput {
- // An observed value of a metric.
- message HyperparameterMetric {
- // The global training step for this metric.
- int64 training_step = 1;
- // The objective value at this training step.
- double objective_value = 2;
- }
- // The trial id for these results.
- string trial_id = 1;
- // The hyperparameters given to this trial.
- map<string, string> hyperparameters = 2;
- // The final objective metric seen for this trial.
- HyperparameterMetric final_metric = 3;
- // All recorded object metrics for this trial.
- repeated HyperparameterMetric all_metrics = 4;
- }
- // Represents results of a training job. Output only.
- message TrainingOutput {
- // The number of hyperparameter tuning trials that completed successfully.
- // Only set for hyperparameter tuning jobs.
- int64 completed_trial_count = 1;
- // Results for individual Hyperparameter trials.
- // Only set for hyperparameter tuning jobs.
- repeated HyperparameterOutput trials = 2;
- // The amount of ML units consumed by the job.
- double consumed_ml_units = 3;
- // Whether this job is a hyperparameter tuning job.
- bool is_hyperparameter_tuning_job = 4;
- }
- // Represents input parameters for a prediction job.
- message PredictionInput {
- // The format used to separate data instances in the source files.
- enum DataFormat {
- // Unspecified format.
- DATA_FORMAT_UNSPECIFIED = 0;
- // The source file is a text file with instances separated by the
- // new-line character.
- TEXT = 1;
- // The source file is a TFRecord file.
- TF_RECORD = 2;
- // The source file is a GZIP-compressed TFRecord file.
- TF_RECORD_GZIP = 3;
- }
- // Required. The model or the version to use for prediction.
- oneof model_version {
- // Use this field if you want to use the default version for the specified
- // model. The string must use the following format:
- //
- // `"projects/<var>[YOUR_PROJECT]</var>/models/<var>[YOUR_MODEL]</var>"`
- string model_name = 1;
- // Use this field if you want to specify a version of the model to use. The
- // string is formatted the same way as `model_version`, with the addition
- // of the version information:
- //
- // `"projects/<var>[YOUR_PROJECT]</var>/models/<var>YOUR_MODEL/versions/<var>[YOUR_VERSION]</var>"`
- string version_name = 2;
- // Use this field if you want to specify a Google Cloud Storage path for
- // the model to use.
- string uri = 9;
- }
- // Required. The format of the input data files.
- DataFormat data_format = 3;
- // Required. The Google Cloud Storage location of the input data files.
- // May contain wildcards.
- repeated string input_paths = 4;
- // Required. The output Google Cloud Storage location.
- string output_path = 5;
- // Optional. The maximum number of workers to be used for parallel processing.
- // Defaults to 10 if not specified.
- int64 max_worker_count = 6;
- // Required. The Google Compute Engine region to run the prediction job in.
- string region = 7;
- // Optional. The Google Cloud ML runtime version to use for this batch
- // prediction. If not set, Google Cloud ML will pick the runtime version used
- // during the CreateVersion request for this model version, or choose the
- // latest stable version when model version information is not available
- // such as when the model is specified by uri.
- string runtime_version = 8;
- }
- // Represents results of a prediction job.
- message PredictionOutput {
- // The output Google Cloud Storage location provided at the job creation time.
- string output_path = 1;
- // The number of generated predictions.
- int64 prediction_count = 2;
- // The number of data instances which resulted in errors.
- int64 error_count = 3;
- // Node hours used by the batch prediction job.
- double node_hours = 4;
- }
- // Represents a training or prediction job.
- message Job {
- // Describes the job state.
- enum State {
- // The job state is unspecified.
- STATE_UNSPECIFIED = 0;
- // The job has been just created and processing has not yet begun.
- QUEUED = 1;
- // The service is preparing to run the job.
- PREPARING = 2;
- // The job is in progress.
- RUNNING = 3;
- // The job completed successfully.
- SUCCEEDED = 4;
- // The job failed.
- // `error_message` should contain the details of the failure.
- FAILED = 5;
- // The job is being cancelled.
- // `error_message` should describe the reason for the cancellation.
- CANCELLING = 6;
- // The job has been cancelled.
- // `error_message` should describe the reason for the cancellation.
- CANCELLED = 7;
- }
- // Required. The user-specified id of the job.
- string job_id = 1;
- // Required. Parameters to create a job.
- oneof input {
- // Input parameters to create a training job.
- TrainingInput training_input = 2;
- // Input parameters to create a prediction job.
- PredictionInput prediction_input = 3;
- }
- // Output only. When the job was created.
- google.protobuf.Timestamp create_time = 4;
- // Output only. When the job processing was started.
- google.protobuf.Timestamp start_time = 5;
- // Output only. When the job processing was completed.
- google.protobuf.Timestamp end_time = 6;
- // Output only. The detailed state of a job.
- State state = 7;
- // Output only. The details of a failure or a cancellation.
- string error_message = 8;
- // Output only. The current result of the job.
- oneof output {
- // The current training job result.
- TrainingOutput training_output = 9;
- // The current prediction job result.
- PredictionOutput prediction_output = 10;
- }
- }
- // Request message for the CreateJob method.
- message CreateJobRequest {
- // Required. The project name.
- //
- // Authorization: requires `Editor` role on the specified project.
- string parent = 1;
- // Required. The job to create.
- Job job = 2;
- }
- // Request message for the ListJobs method.
- message ListJobsRequest {
- // Required. The name of the project for which to list jobs.
- //
- // Authorization: requires `Viewer` role on the specified project.
- string parent = 1;
- // Optional. Specifies the subset of jobs to retrieve.
- string filter = 2;
- // Optional. A page token to request the next page of results.
- //
- // You get the token from the `next_page_token` field of the response from
- // the previous call.
- string page_token = 4;
- // Optional. The number of jobs to retrieve per "page" of results. If there
- // are more remaining results than this number, the response message will
- // contain a valid value in the `next_page_token` field.
- //
- // The default value is 20, and the maximum page size is 100.
- int32 page_size = 5;
- }
- // Response message for the ListJobs method.
- message ListJobsResponse {
- // The list of jobs.
- repeated Job jobs = 1;
- // Optional. Pass this token as the `page_token` field of the request for a
- // subsequent call.
- string next_page_token = 2;
- }
- // Request message for the GetJob method.
- message GetJobRequest {
- // Required. The name of the job to get the description of.
- //
- // Authorization: requires `Viewer` role on the parent project.
- string name = 1;
- }
- // Request message for the CancelJob method.
- message CancelJobRequest {
- // Required. The name of the job to cancel.
- //
- // Authorization: requires `Editor` role on the parent project.
- string name = 1;
- }
|