Add API to automation to get word start and end indices for string.

This change hooks into AXTextUtils::GetWordBoundariesForString and exposes this information in automation. Bug: 948700 Change-Id: I8fe5852cf1ce5e1eeb02e986d4524d44095d054f Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1642390 Reviewed-by: Devlin <rdevlin.cronin@chromium.org> Reviewed-by: Nektarios Paisios <nektar@chromium.org> Commit-Queue: Akihiro Ota <akihiroota@chromium.org> Cr-Commit-Position: refs/heads/master@{#668972}
Denger-Network · Jun 13, 2019 · b747ffb · b747ffb
1 parent 9e04a5e
commit b747ffb
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 0 deletions.
diff --git a/chrome/browser/extensions/api/automation/automation_apitest.cc b/chrome/browser/extensions/api/automation/automation_apitest.cc
@@ -380,6 +380,13 @@ IN_PROC_BROWSER_TEST_F(AutomationApiTest, HitTest) {
       << message_;
 }
 
+IN_PROC_BROWSER_TEST_F(AutomationApiTest, WordBoundaries) {
+  StartEmbeddedTestServer();
+  ASSERT_TRUE(
+      RunExtensionSubtest("automation/tests/tabs", "word_boundaries.html"))
+      << message_;
+}
+
 class AutomationApiTestWithLanguageDetection : public AutomationApiTest {
  protected:
   void SetUpCommandLine(base::CommandLine* command_line) override {

diff --git a/chrome/test/data/extensions/api_test/automation/sites/word_boundaries.html b/chrome/test/data/extensions/api_test/automation/sites/word_boundaries.html
@@ -0,0 +1,14 @@
+<!--
+ * Copyright 2019 The Chromium Authors. All rights reserved.  Use of this
+ * source code is governed by a BSD-style license that can be found in the
+ * LICENSE file.
+-->
+<html>
+<head>
+<title>Automation Tests - Word Boundaries</title>
+</head>
+<body>
+  <!-- Used to test wordBoundariesForString. -->
+  <div role="button">Example text for testing purposes</div>
+</body>
+</html>
diff --git a/chrome/test/data/extensions/api_test/automation/tests/tabs/word_boundaries.html b/chrome/test/data/extensions/api_test/automation/tests/tabs/word_boundaries.html
@@ -0,0 +1,7 @@
+<!--
+ * Copyright 2019 The Chromium Authors. All rights reserved.  Use of this
+ * source code is governed by a BSD-style license that can be found in the
+ * LICENSE file.
+-->
+<script src="common.js"></script>
+<script src="word_boundaries.js"></script>
diff --git a/chrome/test/data/extensions/api_test/automation/tests/tabs/word_boundaries.js b/chrome/test/data/extensions/api_test/automation/tests/tabs/word_boundaries.js
@@ -0,0 +1,24 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+var allTests = [
+  function testWordStartAndEndOffsets() {
+    var node = rootNode.find(
+        { attributes: { name: 'Example text for testing purposes' } });
+    var expectedWordStarts = [0, 8, 13, 17, 25];
+    var expectedWordEnds = [7, 12, 16, 24, 33];
+    var wordStarts = node.wordStartOffsets();
+    var wordEnds = node.wordEndOffsets();
+    assertEq(expectedWordStarts.length, wordStarts.length);
+    assertEq(expectedWordEnds.length, wordEnds.length);
+    assertEq(wordStarts.length, wordEnds.length);
+    for (var i = 0; i < expectedWordStarts.length; ++i){
+        assertEq(expectedWordStarts[i], wordStarts[i]);
+        assertEq(expectedWordEnds[i], wordEnds[i]);
+    }
+    chrome.test.succeed();
+  }
+];
+
+setUpAndRunTests(allTests, 'word_boundaries.html');
diff --git a/extensions/common/api/automation.idl b/extensions/common/api/automation.idl
@@ -589,6 +589,16 @@
     long[]? wordStarts;
     long[]? wordEnds;
 
+    // The start index of each word within the node's name. This is different
+    // from wordStarts because it is not restricted to inline text boxes and can
+    // be used for any type of element.
+    long[]? wordStartOffsets;
+
+    // The end index of each word within the node's name. This is different
+    // from wordEnds because it is not restricted to inline text boxes and can
+    // be used for any type of element.
+    long[]? wordEndOffsets;
+
     // The nodes, if any, which this node is specified to control via
     // <a href="http://www.w3.org/TR/wai-aria/states_and_properties#aria-controls">
     // <code>aria-controls</code></a>.

diff --git a/extensions/renderer/api/automation/automation_internal_custom_bindings.cc b/extensions/renderer/api/automation/automation_internal_custom_bindings.cc
@@ -41,6 +41,7 @@
 #include "ui/accessibility/ax_language_info.h"
 #include "ui/accessibility/ax_node.h"
 #include "ui/accessibility/ax_role_properties.h"
+#include "ui/accessibility/ax_text_utils.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "ui/gfx/geometry/rect_conversions.h"
 
@@ -780,6 +781,22 @@ void AutomationInternalCustomBindings::AddRoutes() {
         response.Set("nodeIds", child_ids);
         result.Set(response.Build());
       });
+  RouteNodeIDFunction(
+      "GetWordStartOffsets",
+      [](v8::Isolate* isolate, v8::ReturnValue<v8::Value> result,
+         AutomationAXTreeWrapper* tree_wrapper, ui::AXNode* node) {
+        std::vector<int> word_starts = ui::GetWordStartOffsets(
+            node->GetString16Attribute(ax::mojom::StringAttribute::kName));
+        result.Set(gin::ConvertToV8(isolate, word_starts));
+      });
+  RouteNodeIDFunction(
+      "GetWordEndOffsets",
+      [](v8::Isolate* isolate, v8::ReturnValue<v8::Value> result,
+         AutomationAXTreeWrapper* tree_wrapper, ui::AXNode* node) {
+        std::vector<int> word_ends = ui::GetWordEndOffsets(
+            node->GetString16Attribute(ax::mojom::StringAttribute::kName));
+        result.Set(gin::ConvertToV8(isolate, word_ends));
+      });
 
   // Bindings that take a Tree ID and Node ID and string attribute name
   // and return a property of the node.

diff --git a/extensions/renderer/resources/automation/automation_node.js b/extensions/renderer/resources/automation/automation_node.js
@@ -466,6 +466,20 @@ var GetDetectedLanguage = natives.GetDetectedLanguage;
 var GetLanguageAnnotationForStringAttribute =
     natives.GetLanguageAnnotationForStringAttribute;
 
+/**
+ * @param {string} axTreeID The id of the accessibility tree.
+ * @param {number} nodeID The id of a node.
+ * @return {!Array<number>}
+ */
+var GetWordStartOffsets = natives.GetWordStartOffsets;
+
+/**
+ * @param {string} axTreeID The id of the accessibility tree.
+ * @param {number} nodeID The id of a node.
+ * @return {!Array<number>}
+ */
+var GetWordEndOffsets = natives.GetWordEndOffsets;
+
 var logging = requireNative('logging');
 var utils = require('utils');
 
@@ -858,6 +872,14 @@ AutomationNodeImpl.prototype = {
       return impl.get(info.nodeId);
   },
 
+  wordStartOffsets: function() {
+    return GetWordStartOffsets(this.treeID, this.id);
+  },
+
+  wordEndOffsets: function() {
+    return GetWordEndOffsets(this.treeID, this.id);
+  },
+
   addEventListener: function(eventType, callback, capture) {
     this.removeEventListener(eventType, callback);
     if (!this.listeners[eventType])
@@ -1679,6 +1701,8 @@ utils.expose(AutomationNode, AutomationNodeImpl, {
     'toString',
     'boundsForRange',
     'languageAnnotationForStringAttribute',
+    'wordStartOffsets',
+    'wordEndOffsets',
   ],
   readonly: $Array.concat(
       publicAttributes,

diff --git a/third_party/closure_compiler/externs/automation.js b/third_party/closure_compiler/externs/automation.js
@@ -651,6 +651,19 @@ chrome.automation.AutomationNode.prototype.wordStarts;
  */
 chrome.automation.AutomationNode.prototype.wordEnds;
 
+/**
+ * The start and end index of each word within the node's name. Different from wordStarts and wordEnds because they're not restricted to inline text boxes and can be used for any type of element.
+ * @type {(!Array<number>|undefined)}
+ * @see https://developer.chrome.com/extensions/automation#type-wordStartOffsets
+ */
+chrome.automation.AutomationNode.prototype.wordStartOffsets;
+
+/**
+ * @type {(!Array<number>|undefined)}
+ * @see https://developer.chrome.com/extensions/automation#type-wordEndOffsets
+ */
+chrome.automation.AutomationNode.prototype.wordEndOffsets;
+
 /**
  * The nodes, if any, which this node is specified to control via <a href="http://www.w3.org/TR/wai-aria/states_and_properties#aria-controls"> <code>aria-controls</code></a>.
  * @type {(!Array<!chrome.automation.AutomationNode>|undefined)}