image_annotator.proto 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614
  1. // Copyright 2019 Google LLC.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. //
  15. syntax = "proto3";
  16. package google.cloud.vision.v1p1beta1;
  17. import "google/api/annotations.proto";
  18. import "google/api/client.proto";
  19. import "google/api/field_behavior.proto";
  20. import "google/cloud/vision/v1p1beta1/geometry.proto";
  21. import "google/cloud/vision/v1p1beta1/text_annotation.proto";
  22. import "google/cloud/vision/v1p1beta1/web_detection.proto";
  23. import "google/rpc/status.proto";
  24. import "google/type/color.proto";
  25. import "google/type/latlng.proto";
  26. option cc_enable_arenas = true;
  27. option go_package = "google.golang.org/genproto/googleapis/cloud/vision/v1p1beta1;vision";
  28. option java_multiple_files = true;
  29. option java_outer_classname = "ImageAnnotatorProto";
  30. option java_package = "com.google.cloud.vision.v1p1beta1";
  31. // Service that performs Google Cloud Vision API detection tasks over client
  32. // images, such as face, landmark, logo, label, and text detection. The
  33. // ImageAnnotator service returns detected entities from the images.
  34. service ImageAnnotator {
  35. option (google.api.default_host) = "vision.googleapis.com";
  36. option (google.api.oauth_scopes) =
  37. "https://www.googleapis.com/auth/cloud-platform,"
  38. "https://www.googleapis.com/auth/cloud-vision";
  39. // Run image detection and annotation for a batch of images.
  40. rpc BatchAnnotateImages(BatchAnnotateImagesRequest)
  41. returns (BatchAnnotateImagesResponse) {
  42. option (google.api.http) = {
  43. post: "/v1p1beta1/images:annotate"
  44. body: "*"
  45. };
  46. option (google.api.method_signature) = "requests";
  47. }
  48. }
  49. // Users describe the type of Google Cloud Vision API tasks to perform over
  50. // images by using *Feature*s. Each Feature indicates a type of image
  51. // detection task to perform. Features encode the Cloud Vision API
  52. // vertical to operate on and the number of top-scoring results to return.
  53. message Feature {
  54. // Type of image feature.
  55. enum Type {
  56. // Unspecified feature type.
  57. TYPE_UNSPECIFIED = 0;
  58. // Run face detection.
  59. FACE_DETECTION = 1;
  60. // Run landmark detection.
  61. LANDMARK_DETECTION = 2;
  62. // Run logo detection.
  63. LOGO_DETECTION = 3;
  64. // Run label detection.
  65. LABEL_DETECTION = 4;
  66. // Run OCR.
  67. TEXT_DETECTION = 5;
  68. // Run dense text document OCR. Takes precedence when both
  69. // DOCUMENT_TEXT_DETECTION and TEXT_DETECTION are present.
  70. DOCUMENT_TEXT_DETECTION = 11;
  71. // Run computer vision models to compute image safe-search properties.
  72. SAFE_SEARCH_DETECTION = 6;
  73. // Compute a set of image properties, such as the image's dominant colors.
  74. IMAGE_PROPERTIES = 7;
  75. // Run crop hints.
  76. CROP_HINTS = 9;
  77. // Run web detection.
  78. WEB_DETECTION = 10;
  79. }
  80. // The feature type.
  81. Type type = 1;
  82. // Maximum number of results of this type.
  83. int32 max_results = 2;
  84. // Model to use for the feature.
  85. // Supported values: "builtin/stable" (the default if unset) and
  86. // "builtin/latest".
  87. string model = 3;
  88. }
  89. // External image source (Google Cloud Storage image location).
  90. message ImageSource {
  91. // NOTE: For new code `image_uri` below is preferred.
  92. // Google Cloud Storage image URI, which must be in the following form:
  93. // `gs://bucket_name/object_name` (for details, see
  94. // [Google Cloud Storage Request
  95. // URIs](https://cloud.google.com/storage/docs/reference-uris)).
  96. // NOTE: Cloud Storage object versioning is not supported.
  97. string gcs_image_uri = 1;
  98. // Image URI which supports:
  99. // 1) Google Cloud Storage image URI, which must be in the following form:
  100. // `gs://bucket_name/object_name` (for details, see
  101. // [Google Cloud Storage Request
  102. // URIs](https://cloud.google.com/storage/docs/reference-uris)).
  103. // NOTE: Cloud Storage object versioning is not supported.
  104. // 2) Publicly accessible image HTTP/HTTPS URL.
  105. // This is preferred over the legacy `gcs_image_uri` above. When both
  106. // `gcs_image_uri` and `image_uri` are specified, `image_uri` takes
  107. // precedence.
  108. string image_uri = 2;
  109. }
  110. // Client image to perform Google Cloud Vision API tasks over.
  111. message Image {
  112. // Image content, represented as a stream of bytes.
  113. // Note: as with all `bytes` fields, protobuffers use a pure binary
  114. // representation, whereas JSON representations use base64.
  115. bytes content = 1;
  116. // Google Cloud Storage image location. If both `content` and `source`
  117. // are provided for an image, `content` takes precedence and is
  118. // used to perform the image annotation request.
  119. ImageSource source = 2;
  120. }
  121. // A face annotation object contains the results of face detection.
  122. message FaceAnnotation {
  123. // A face-specific landmark (for example, a face feature).
  124. message Landmark {
  125. // Face landmark (feature) type.
  126. // Left and right are defined from the vantage of the viewer of the image
  127. // without considering mirror projections typical of photos. So, `LEFT_EYE`,
  128. // typically, is the person's right eye.
  129. enum Type {
  130. // Unknown face landmark detected. Should not be filled.
  131. UNKNOWN_LANDMARK = 0;
  132. // Left eye.
  133. LEFT_EYE = 1;
  134. // Right eye.
  135. RIGHT_EYE = 2;
  136. // Left of left eyebrow.
  137. LEFT_OF_LEFT_EYEBROW = 3;
  138. // Right of left eyebrow.
  139. RIGHT_OF_LEFT_EYEBROW = 4;
  140. // Left of right eyebrow.
  141. LEFT_OF_RIGHT_EYEBROW = 5;
  142. // Right of right eyebrow.
  143. RIGHT_OF_RIGHT_EYEBROW = 6;
  144. // Midpoint between eyes.
  145. MIDPOINT_BETWEEN_EYES = 7;
  146. // Nose tip.
  147. NOSE_TIP = 8;
  148. // Upper lip.
  149. UPPER_LIP = 9;
  150. // Lower lip.
  151. LOWER_LIP = 10;
  152. // Mouth left.
  153. MOUTH_LEFT = 11;
  154. // Mouth right.
  155. MOUTH_RIGHT = 12;
  156. // Mouth center.
  157. MOUTH_CENTER = 13;
  158. // Nose, bottom right.
  159. NOSE_BOTTOM_RIGHT = 14;
  160. // Nose, bottom left.
  161. NOSE_BOTTOM_LEFT = 15;
  162. // Nose, bottom center.
  163. NOSE_BOTTOM_CENTER = 16;
  164. // Left eye, top boundary.
  165. LEFT_EYE_TOP_BOUNDARY = 17;
  166. // Left eye, right corner.
  167. LEFT_EYE_RIGHT_CORNER = 18;
  168. // Left eye, bottom boundary.
  169. LEFT_EYE_BOTTOM_BOUNDARY = 19;
  170. // Left eye, left corner.
  171. LEFT_EYE_LEFT_CORNER = 20;
  172. // Right eye, top boundary.
  173. RIGHT_EYE_TOP_BOUNDARY = 21;
  174. // Right eye, right corner.
  175. RIGHT_EYE_RIGHT_CORNER = 22;
  176. // Right eye, bottom boundary.
  177. RIGHT_EYE_BOTTOM_BOUNDARY = 23;
  178. // Right eye, left corner.
  179. RIGHT_EYE_LEFT_CORNER = 24;
  180. // Left eyebrow, upper midpoint.
  181. LEFT_EYEBROW_UPPER_MIDPOINT = 25;
  182. // Right eyebrow, upper midpoint.
  183. RIGHT_EYEBROW_UPPER_MIDPOINT = 26;
  184. // Left ear tragion.
  185. LEFT_EAR_TRAGION = 27;
  186. // Right ear tragion.
  187. RIGHT_EAR_TRAGION = 28;
  188. // Left eye pupil.
  189. LEFT_EYE_PUPIL = 29;
  190. // Right eye pupil.
  191. RIGHT_EYE_PUPIL = 30;
  192. // Forehead glabella.
  193. FOREHEAD_GLABELLA = 31;
  194. // Chin gnathion.
  195. CHIN_GNATHION = 32;
  196. // Chin left gonion.
  197. CHIN_LEFT_GONION = 33;
  198. // Chin right gonion.
  199. CHIN_RIGHT_GONION = 34;
  200. }
  201. // Face landmark type.
  202. Type type = 3;
  203. // Face landmark position.
  204. Position position = 4;
  205. }
  206. // The bounding polygon around the face. The coordinates of the bounding box
  207. // are in the original image's scale, as returned in `ImageParams`.
  208. // The bounding box is computed to "frame" the face in accordance with human
  209. // expectations. It is based on the landmarker results.
  210. // Note that one or more x and/or y coordinates may not be generated in the
  211. // `BoundingPoly` (the polygon will be unbounded) if only a partial face
  212. // appears in the image to be annotated.
  213. BoundingPoly bounding_poly = 1;
  214. // The `fd_bounding_poly` bounding polygon is tighter than the
  215. // `boundingPoly`, and encloses only the skin part of the face. Typically, it
  216. // is used to eliminate the face from any image analysis that detects the
  217. // "amount of skin" visible in an image. It is not based on the
  218. // landmarker results, only on the initial face detection, hence
  219. // the <code>fd</code> (face detection) prefix.
  220. BoundingPoly fd_bounding_poly = 2;
  221. // Detected face landmarks.
  222. repeated Landmark landmarks = 3;
  223. // Roll angle, which indicates the amount of clockwise/anti-clockwise rotation
  224. // of the face relative to the image vertical about the axis perpendicular to
  225. // the face. Range [-180,180].
  226. float roll_angle = 4;
  227. // Yaw angle, which indicates the leftward/rightward angle that the face is
  228. // pointing relative to the vertical plane perpendicular to the image. Range
  229. // [-180,180].
  230. float pan_angle = 5;
  231. // Pitch angle, which indicates the upwards/downwards angle that the face is
  232. // pointing relative to the image's horizontal plane. Range [-180,180].
  233. float tilt_angle = 6;
  234. // Detection confidence. Range [0, 1].
  235. float detection_confidence = 7;
  236. // Face landmarking confidence. Range [0, 1].
  237. float landmarking_confidence = 8;
  238. // Joy likelihood.
  239. Likelihood joy_likelihood = 9;
  240. // Sorrow likelihood.
  241. Likelihood sorrow_likelihood = 10;
  242. // Anger likelihood.
  243. Likelihood anger_likelihood = 11;
  244. // Surprise likelihood.
  245. Likelihood surprise_likelihood = 12;
  246. // Under-exposed likelihood.
  247. Likelihood under_exposed_likelihood = 13;
  248. // Blurred likelihood.
  249. Likelihood blurred_likelihood = 14;
  250. // Headwear likelihood.
  251. Likelihood headwear_likelihood = 15;
  252. }
  253. // Detected entity location information.
  254. message LocationInfo {
  255. // lat/long location coordinates.
  256. google.type.LatLng lat_lng = 1;
  257. }
  258. // A `Property` consists of a user-supplied name/value pair.
  259. message Property {
  260. // Name of the property.
  261. string name = 1;
  262. // Value of the property.
  263. string value = 2;
  264. // Value of numeric properties.
  265. uint64 uint64_value = 3;
  266. }
  267. // Set of detected entity features.
  268. message EntityAnnotation {
  269. // Opaque entity ID. Some IDs may be available in
  270. // [Google Knowledge Graph Search
  271. // API](https://developers.google.com/knowledge-graph/).
  272. string mid = 1;
  273. // The language code for the locale in which the entity textual
  274. // `description` is expressed.
  275. string locale = 2;
  276. // Entity textual description, expressed in its `locale` language.
  277. string description = 3;
  278. // Overall score of the result. Range [0, 1].
  279. float score = 4;
  280. // The accuracy of the entity detection in an image.
  281. // For example, for an image in which the "Eiffel Tower" entity is detected,
  282. // this field represents the confidence that there is a tower in the query
  283. // image. Range [0, 1].
  284. float confidence = 5;
  285. // The relevancy of the ICA (Image Content Annotation) label to the
  286. // image. For example, the relevancy of "tower" is likely higher to an image
  287. // containing the detected "Eiffel Tower" than to an image containing a
  288. // detected distant towering building, even though the confidence that
  289. // there is a tower in each image may be the same. Range [0, 1].
  290. float topicality = 6;
  291. // Image region to which this entity belongs. Not produced
  292. // for `LABEL_DETECTION` features.
  293. BoundingPoly bounding_poly = 7;
  294. // The location information for the detected entity. Multiple
  295. // `LocationInfo` elements can be present because one location may
  296. // indicate the location of the scene in the image, and another location
  297. // may indicate the location of the place where the image was taken.
  298. // Location information is usually present for landmarks.
  299. repeated LocationInfo locations = 8;
  300. // Some entities may have optional user-supplied `Property` (name/value)
  301. // fields, such a score or string that qualifies the entity.
  302. repeated Property properties = 9;
  303. }
  304. // Set of features pertaining to the image, computed by computer vision
  305. // methods over safe-search verticals (for example, adult, spoof, medical,
  306. // violence).
  307. message SafeSearchAnnotation {
  308. // Represents the adult content likelihood for the image. Adult content may
  309. // contain elements such as nudity, pornographic images or cartoons, or
  310. // sexual activities.
  311. Likelihood adult = 1;
  312. // Spoof likelihood. The likelihood that an modification
  313. // was made to the image's canonical version to make it appear
  314. // funny or offensive.
  315. Likelihood spoof = 2;
  316. // Likelihood that this is a medical image.
  317. Likelihood medical = 3;
  318. // Likelihood that this image contains violent content.
  319. Likelihood violence = 4;
  320. // Likelihood that the request image contains racy content. Racy content may
  321. // include (but is not limited to) skimpy or sheer clothing, strategically
  322. // covered nudity, lewd or provocative poses, or close-ups of sensitive
  323. // body areas.
  324. Likelihood racy = 9;
  325. }
  326. // Rectangle determined by min and max `LatLng` pairs.
  327. message LatLongRect {
  328. // Min lat/long pair.
  329. google.type.LatLng min_lat_lng = 1;
  330. // Max lat/long pair.
  331. google.type.LatLng max_lat_lng = 2;
  332. }
  333. // Color information consists of RGB channels, score, and the fraction of
  334. // the image that the color occupies in the image.
  335. message ColorInfo {
  336. // RGB components of the color.
  337. google.type.Color color = 1;
  338. // Image-specific score for this color. Value in range [0, 1].
  339. float score = 2;
  340. // The fraction of pixels the color occupies in the image.
  341. // Value in range [0, 1].
  342. float pixel_fraction = 3;
  343. }
  344. // Set of dominant colors and their corresponding scores.
  345. message DominantColorsAnnotation {
  346. // RGB color values with their score and pixel fraction.
  347. repeated ColorInfo colors = 1;
  348. }
  349. // Stores image properties, such as dominant colors.
  350. message ImageProperties {
  351. // If present, dominant colors completed successfully.
  352. DominantColorsAnnotation dominant_colors = 1;
  353. }
  354. // Single crop hint that is used to generate a new crop when serving an image.
  355. message CropHint {
  356. // The bounding polygon for the crop region. The coordinates of the bounding
  357. // box are in the original image's scale, as returned in `ImageParams`.
  358. BoundingPoly bounding_poly = 1;
  359. // Confidence of this being a salient region. Range [0, 1].
  360. float confidence = 2;
  361. // Fraction of importance of this salient region with respect to the original
  362. // image.
  363. float importance_fraction = 3;
  364. }
  365. // Set of crop hints that are used to generate new crops when serving images.
  366. message CropHintsAnnotation {
  367. // Crop hint results.
  368. repeated CropHint crop_hints = 1;
  369. }
  370. // Parameters for crop hints annotation request.
  371. message CropHintsParams {
  372. // Aspect ratios in floats, representing the ratio of the width to the height
  373. // of the image. For example, if the desired aspect ratio is 4/3, the
  374. // corresponding float value should be 1.33333. If not specified, the
  375. // best possible crop is returned. The number of provided aspect ratios is
  376. // limited to a maximum of 16; any aspect ratios provided after the 16th are
  377. // ignored.
  378. repeated float aspect_ratios = 1;
  379. }
  380. // Parameters for web detection request.
  381. message WebDetectionParams {
  382. // Whether to include results derived from the geo information in the image.
  383. bool include_geo_results = 2;
  384. }
  385. // Parameters for text detections. This is used to control TEXT_DETECTION and
  386. // DOCUMENT_TEXT_DETECTION features.
  387. message TextDetectionParams {
  388. // By default, Cloud Vision API only includes confidence score for
  389. // DOCUMENT_TEXT_DETECTION result. Set the flag to true to include confidence
  390. // score for TEXT_DETECTION as well.
  391. bool enable_text_detection_confidence_score = 9;
  392. }
  393. // Image context and/or feature-specific parameters.
  394. message ImageContext {
  395. // lat/long rectangle that specifies the location of the image.
  396. LatLongRect lat_long_rect = 1;
  397. // List of languages to use for TEXT_DETECTION. In most cases, an empty value
  398. // yields the best results since it enables automatic language detection. For
  399. // languages based on the Latin alphabet, setting `language_hints` is not
  400. // needed. In rare cases, when the language of the text in the image is known,
  401. // setting a hint will help get better results (although it will be a
  402. // significant hindrance if the hint is wrong). Text detection returns an
  403. // error if one or more of the specified languages is not one of the
  404. // [supported languages](https://cloud.google.com/vision/docs/languages).
  405. repeated string language_hints = 2;
  406. // Parameters for crop hints annotation request.
  407. CropHintsParams crop_hints_params = 4;
  408. // Parameters for web detection.
  409. WebDetectionParams web_detection_params = 6;
  410. // Parameters for text detection and document text detection.
  411. TextDetectionParams text_detection_params = 12;
  412. }
  413. // Request for performing Google Cloud Vision API tasks over a user-provided
  414. // image, with user-requested features.
  415. message AnnotateImageRequest {
  416. // The image to be processed.
  417. Image image = 1;
  418. // Requested features.
  419. repeated Feature features = 2;
  420. // Additional context that may accompany the image.
  421. ImageContext image_context = 3;
  422. }
  423. // Response to an image annotation request.
  424. message AnnotateImageResponse {
  425. // If present, face detection has completed successfully.
  426. repeated FaceAnnotation face_annotations = 1;
  427. // If present, landmark detection has completed successfully.
  428. repeated EntityAnnotation landmark_annotations = 2;
  429. // If present, logo detection has completed successfully.
  430. repeated EntityAnnotation logo_annotations = 3;
  431. // If present, label detection has completed successfully.
  432. repeated EntityAnnotation label_annotations = 4;
  433. // If present, text (OCR) detection has completed successfully.
  434. repeated EntityAnnotation text_annotations = 5;
  435. // If present, text (OCR) detection or document (OCR) text detection has
  436. // completed successfully.
  437. // This annotation provides the structural hierarchy for the OCR detected
  438. // text.
  439. TextAnnotation full_text_annotation = 12;
  440. // If present, safe-search annotation has completed successfully.
  441. SafeSearchAnnotation safe_search_annotation = 6;
  442. // If present, image properties were extracted successfully.
  443. ImageProperties image_properties_annotation = 8;
  444. // If present, crop hints have completed successfully.
  445. CropHintsAnnotation crop_hints_annotation = 11;
  446. // If present, web detection has completed successfully.
  447. WebDetection web_detection = 13;
  448. // If set, represents the error message for the operation.
  449. // Note that filled-in image annotations are guaranteed to be
  450. // correct, even when `error` is set.
  451. google.rpc.Status error = 9;
  452. }
  453. // Multiple image annotation requests are batched into a single service call.
  454. message BatchAnnotateImagesRequest {
  455. // Required. Individual image annotation requests for this batch.
  456. repeated AnnotateImageRequest requests = 1 [(google.api.field_behavior) = REQUIRED];
  457. }
  458. // Response to a batch image annotation request.
  459. message BatchAnnotateImagesResponse {
  460. // Individual responses to image annotation requests within the batch.
  461. repeated AnnotateImageResponse responses = 1;
  462. }
  463. // A bucketized representation of likelihood, which is intended to give clients
  464. // highly stable results across model upgrades.
  465. enum Likelihood {
  466. // Unknown likelihood.
  467. UNKNOWN = 0;
  468. // It is very unlikely that the image belongs to the specified vertical.
  469. VERY_UNLIKELY = 1;
  470. // It is unlikely that the image belongs to the specified vertical.
  471. UNLIKELY = 2;
  472. // It is possible that the image belongs to the specified vertical.
  473. POSSIBLE = 3;
  474. // It is likely that the image belongs to the specified vertical.
  475. LIKELY = 4;
  476. // It is very likely that the image belongs to the specified vertical.
  477. VERY_LIKELY = 5;
  478. }