Skip to content

Commit

Permalink
Add mouse-related APIs to autotestPrivate
Browse files Browse the repository at this point in the history 
This CL adds getting and setting the mouse state to autotestPrivate
API. Although Tast / testing infra should have the feature to
cause mouse events through device files, it turns out not quite
helpful for the use of UI tests; the new APIs addresses to the
situation; obtaining the current mouse status, and causing the mouse
events exactly at a location.

Bug: 1019000
Test: with Tast
Change-Id: Iafacca190e057653565a685794cf3833a2411a2b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1896385
Reviewed-by: Steven Bennetts <stevenjb@chromium.org>
Reviewed-by: Mitsuru Oshima <oshima@chromium.org>
Commit-Queue: Jun Mukai <mukai@chromium.org>
Cr-Commit-Position: refs/heads/master@{#714986}
  • Loading branch information
@jmuk
jmuk authored and Commit Bot committed Nov 13, 2019
1 parent c71befa commit 7077319
Show file tree
Hide file tree
Showing 5 changed files with 366 additions and 0 deletions.
264 changes: 264 additions & 0 deletions chrome/browser/chromeos/extensions/autotest_private/autotest_private_api.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,20 +128,25 @@
#include "mojo/public/cpp/bindings/remote.h"
#include "net/base/filename_util.h"
#include "ui/aura/client/aura_constants.h"
#include "ui/aura/env.h"
#include "ui/aura/window.h"
#include "ui/aura/window_observer.h"
#include "ui/aura/window_tree_host.h"
#include "ui/base/accelerators/accelerator_history.h"
#include "ui/base/clipboard/clipboard.h"
#include "ui/base/clipboard/scoped_clipboard_writer.h"
#include "ui/base/ime/ime_bridge.h"
#include "ui/base/ui_base_features.h"
#include "ui/display/display.h"
#include "ui/display/screen.h"
#include "ui/events/event_constants.h"
#include "ui/message_center/message_center.h"
#include "ui/message_center/notification_list.h"
#include "ui/message_center/public/cpp/notification.h"
#include "ui/message_center/public/cpp/notification_types.h"
#include "ui/views/widget/widget.h"
#include "ui/wm/core/coordinate_conversion.h"
#include "ui/wm/core/cursor_manager.h"
#include "ui/wm/core/window_util.h"
#include "ui/wm/public/activation_client.h"
#include "url/gurl.h"
Expand Down Expand Up @@ -585,6 +590,27 @@ bool IsFrameVisible(views::Widget* widget) {
return frame_view && frame_view->GetEnabled() && frame_view->GetVisible();
}

void ConvertPointToHost(aura::Window* root_window, gfx::PointF* location) {
gfx::Point3F transformed_location_in_root(*location);
root_window->GetHost()->GetRootTransform().TransformPoint(
&transformed_location_in_root);
*location = transformed_location_in_root.AsPointF();
}

int GetMouseEventFlags(api::autotest_private::MouseButton button) {
switch (button) {
case api::autotest_private::MOUSE_BUTTON_LEFT:
return ui::EF_LEFT_MOUSE_BUTTON;
case api::autotest_private::MOUSE_BUTTON_RIGHT:
return ui::EF_RIGHT_MOUSE_BUTTON;
case api::autotest_private::MOUSE_BUTTON_MIDDLE:
return ui::EF_MIDDLE_MOUSE_BUTTON;
default:
NOTREACHED();
}
return ui::EF_NONE;
}

} // namespace

class WindowStateChangeObserver : public aura::WindowObserver {
Expand Down Expand Up @@ -624,6 +650,65 @@ class WindowStateChangeObserver : public aura::WindowObserver {
DISALLOW_COPY_AND_ASSIGN(WindowStateChangeObserver);
};

class EventGenerator {
public:
EventGenerator(aura::WindowTreeHost* host, base::OnceClosure closure)
: host_(host), closure_(std::move(closure)), weak_ptr_factory_(this) {}
~EventGenerator() = default;

void ScheduleMouseEvent(ui::EventType type,
gfx::PointF location_in_host,
int flags) {
int last_flags = (events_.empty())
? aura::Env::GetInstance()->mouse_button_flags()
: events_.back()->flags();
events_.push_back(std::make_unique<ui::MouseEvent>(
type, location_in_host, location_in_host, base::TimeTicks::Now(), flags,
flags ^ last_flags));
}

void Run() {
last_event_timestamp_ = base::TimeTicks::Now();
SendEvent();
}

base::TimeDelta GetInterval() const {
return base::TimeDelta::FromSeconds(1) /
host_->compositor()->refresh_rate();
}

private:
void SendEvent() {
if (events_.empty()) {
std::move(closure_).Run();
return;
}
auto event = ui::Event::Clone(*events_.front());
events_.pop_front();
{
ui::Event::DispatcherApi api(event.get());
api.set_time_stamp(base::TimeTicks::Now());
}
host_->SendEventToSink(event.get());
base::TimeTicks current_timestamp = base::TimeTicks::Now();
base::TimeDelta interval =
last_event_timestamp_ + GetInterval() - current_timestamp;
base::SequencedTaskRunnerHandle::Get()->PostDelayedTask(
FROM_HERE,
base::BindOnce(&EventGenerator::SendEvent,
weak_ptr_factory_.GetWeakPtr()),
interval);
last_event_timestamp_ = current_timestamp;
}

base::TimeTicks last_event_timestamp_;
aura::WindowTreeHost* const host_;
base::OnceClosure closure_;
std::deque<std::unique_ptr<ui::Event>> events_;

base::WeakPtrFactory<EventGenerator> weak_ptr_factory_;
};

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateInitializeEventsFunction
///////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -3521,6 +3606,185 @@ void AutotestPrivateRemoveActiveDeskFunction::OnAnimationComplete() {
Respond(OneArgument(std::make_unique<base::Value>(true)));
}

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateMouseClickFunction
///////////////////////////////////////////////////////////////////////////////

AutotestPrivateMouseClickFunction::AutotestPrivateMouseClickFunction() =
default;

AutotestPrivateMouseClickFunction::~AutotestPrivateMouseClickFunction() =
default;

ExtensionFunction::ResponseAction AutotestPrivateMouseClickFunction::Run() {
std::unique_ptr<api::autotest_private::MouseClick::Params> params(
api::autotest_private::MouseClick::Params::Create(*args_));
EXTENSION_FUNCTION_VALIDATE(params);

auto* env = aura::Env::GetInstance();
if (env->mouse_button_flags() != 0) {
return RespondNow(Error(base::StringPrintf("Already pressed; flags %d",
env->mouse_button_flags())));
}

int64_t display_id = ash::Shell::Get()->cursor_manager()->GetDisplay().id();
auto* root_window = ash::Shell::GetRootWindowForDisplayId(display_id);
if (!root_window)
return RespondNow(Error("Failed to find the root window"));

gfx::PointF location_in_host(env->last_mouse_location().x(),
env->last_mouse_location().y());
wm::ConvertPointFromScreen(root_window, &location_in_host);
ConvertPointToHost(root_window, &location_in_host);

int flags = GetMouseEventFlags(params->button);
event_generator_ = std::make_unique<EventGenerator>(
root_window->GetHost(),
base::BindOnce(&AutotestPrivateMouseClickFunction::Respond, this,
NoArguments()));
event_generator_->ScheduleMouseEvent(ui::ET_MOUSE_PRESSED, location_in_host,
flags);
event_generator_->ScheduleMouseEvent(ui::ET_MOUSE_RELEASED, location_in_host,
0);
event_generator_->Run();

return RespondLater();
}

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateMousePressFunction
///////////////////////////////////////////////////////////////////////////////

AutotestPrivateMousePressFunction::AutotestPrivateMousePressFunction() =
default;
AutotestPrivateMousePressFunction::~AutotestPrivateMousePressFunction() =
default;

ExtensionFunction::ResponseAction AutotestPrivateMousePressFunction::Run() {
std::unique_ptr<api::autotest_private::MousePress::Params> params(
api::autotest_private::MousePress::Params::Create(*args_));
EXTENSION_FUNCTION_VALIDATE(params);

auto* env = aura::Env::GetInstance();
int flags = env->mouse_button_flags() | GetMouseEventFlags(params->button);
if (flags == env->mouse_button_flags())
return RespondNow(NoArguments());

int64_t display_id = ash::Shell::Get()->cursor_manager()->GetDisplay().id();
auto* root_window = ash::Shell::GetRootWindowForDisplayId(display_id);
if (!root_window)
return RespondNow(Error("Failed to find the root window"));

gfx::PointF location_in_host(env->last_mouse_location().x(),
env->last_mouse_location().y());
wm::ConvertPointFromScreen(root_window, &location_in_host);
ConvertPointToHost(root_window, &location_in_host);

event_generator_ = std::make_unique<EventGenerator>(
root_window->GetHost(),
base::BindOnce(&AutotestPrivateMousePressFunction::Respond, this,
NoArguments()));
event_generator_->ScheduleMouseEvent(ui::ET_MOUSE_PRESSED, location_in_host,
flags);
event_generator_->Run();

return RespondLater();
}

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateMouseReleaseFunction
///////////////////////////////////////////////////////////////////////////////

AutotestPrivateMouseReleaseFunction::AutotestPrivateMouseReleaseFunction() =
default;
AutotestPrivateMouseReleaseFunction::~AutotestPrivateMouseReleaseFunction() =
default;

ExtensionFunction::ResponseAction AutotestPrivateMouseReleaseFunction::Run() {
std::unique_ptr<api::autotest_private::MouseRelease::Params> params(
api::autotest_private::MouseRelease::Params::Create(*args_));
EXTENSION_FUNCTION_VALIDATE(params);

auto* env = aura::Env::GetInstance();

int flags = env->mouse_button_flags() & (~GetMouseEventFlags(params->button));
if (flags == env->mouse_button_flags())
return RespondNow(NoArguments());

int64_t display_id = ash::Shell::Get()->cursor_manager()->GetDisplay().id();
auto* root_window = ash::Shell::GetRootWindowForDisplayId(display_id);
if (!root_window)
return RespondNow(Error("Failed to find the root window"));

gfx::PointF location_in_host(env->last_mouse_location().x(),
env->last_mouse_location().y());
wm::ConvertPointFromScreen(root_window, &location_in_host);
ConvertPointToHost(root_window, &location_in_host);

event_generator_ = std::make_unique<EventGenerator>(
root_window->GetHost(),
base::BindOnce(&AutotestPrivateMouseReleaseFunction::Respond, this,
NoArguments()));
event_generator_->ScheduleMouseEvent(ui::ET_MOUSE_RELEASED, location_in_host,
flags);
event_generator_->Run();

return RespondLater();
}

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateMouseMoveFunction
///////////////////////////////////////////////////////////////////////////////

AutotestPrivateMouseMoveFunction::AutotestPrivateMouseMoveFunction() = default;
AutotestPrivateMouseMoveFunction::~AutotestPrivateMouseMoveFunction() = default;
ExtensionFunction::ResponseAction AutotestPrivateMouseMoveFunction::Run() {
std::unique_ptr<api::autotest_private::MouseMove::Params> params(
api::autotest_private::MouseMove::Params::Create(*args_));
EXTENSION_FUNCTION_VALIDATE(params);

int64_t display_id = ash::Shell::Get()->cursor_manager()->GetDisplay().id();
auto* root_window = ash::Shell::GetRootWindowForDisplayId(display_id);
if (!root_window)
return RespondNow(Error("Failed to find the root window"));

auto* host = root_window->GetHost();
const gfx::PointF location_in_root(params->location.x, params->location.y);
gfx::PointF location_in_screen = location_in_root;
wm::ConvertPointToScreen(root_window, &location_in_screen);
auto* env = aura::Env::GetInstance();
const gfx::Point last_mouse_location(env->last_mouse_location());
if (last_mouse_location == gfx::ToFlooredPoint(location_in_screen))
return RespondNow(NoArguments());

gfx::PointF location_in_host = location_in_root;
ConvertPointToHost(root_window, &location_in_host);

event_generator_ = std::make_unique<EventGenerator>(
host, base::BindOnce(&AutotestPrivateMouseMoveFunction::Respond, this,
NoArguments()));
gfx::PointF start_in_host(last_mouse_location.x(), last_mouse_location.y());
wm::ConvertPointFromScreen(root_window, &start_in_host);
ConvertPointToHost(root_window, &start_in_host);

int64_t steps =
std::max(base::TimeDelta::FromMilliseconds(params->duration_in_ms) /
event_generator_->GetInterval(),
static_cast<int64_t>(1));
int flags = env->mouse_button_flags();
ui::EventType type = (flags == 0) ? ui::ET_MOUSE_MOVED : ui::ET_MOUSE_DRAGGED;
for (int64_t i = 1; i <= steps; ++i) {
double progress = double(i) / double(steps);
gfx::PointF point(gfx::Tween::FloatValueBetween(progress, start_in_host.x(),
location_in_host.x()),
gfx::Tween::FloatValueBetween(progress, start_in_host.y(),
location_in_host.y()));
event_generator_->ScheduleMouseEvent(type, point, flags);
}
event_generator_->Run();
return RespondLater();
}

///////////////////////////////////////////////////////////////////////////////
// AutotestPrivateAPI
///////////////////////////////////////////////////////////////////////////////
Expand Down
55 changes: 55 additions & 0 deletions chrome/browser/chromeos/extensions/autotest_private/autotest_private_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ namespace extensions {

class AssistantInteractionHelper;
class WindowStateChangeObserver;
class EventGenerator;

class AutotestPrivateInitializeEventsFunction : public ExtensionFunction {
public:
Expand Down Expand Up @@ -1052,6 +1053,60 @@ class AutotestPrivateRemoveActiveDeskFunction : public ExtensionFunction {
void OnAnimationComplete();
};

class AutotestPrivateMouseClickFunction : public ExtensionFunction {
public:
AutotestPrivateMouseClickFunction();
DECLARE_EXTENSION_FUNCTION("autotestPrivate.mouseClick",
AUTOTESTPRIVATE_MOUSECLICK)

private:
~AutotestPrivateMouseClickFunction() override;
ResponseAction Run() override;

std::unique_ptr<EventGenerator> event_generator_;
};

class AutotestPrivateMousePressFunction : public ExtensionFunction {
public:
AutotestPrivateMousePressFunction();
DECLARE_EXTENSION_FUNCTION("autotestPrivate.mousePress",
AUTOTESTPRIVATE_MOUSEPRESS)

private:
~AutotestPrivateMousePressFunction() override;
ResponseAction Run() override;

std::unique_ptr<EventGenerator> event_generator_;
};

class AutotestPrivateMouseReleaseFunction : public ExtensionFunction {
public:
AutotestPrivateMouseReleaseFunction();
DECLARE_EXTENSION_FUNCTION("autotestPrivate.mouseRelease",
AUTOTESTPRIVATE_MOUSERELEASE)

private:
~AutotestPrivateMouseReleaseFunction() override;
ResponseAction Run() override;

std::unique_ptr<EventGenerator> event_generator_;
};

class AutotestPrivateMouseMoveFunction : public ExtensionFunction {
public:
AutotestPrivateMouseMoveFunction();
DECLARE_EXTENSION_FUNCTION("autotestPrivate.mouseMove",
AUTOTESTPRIVATE_MOUSEMOVE)

private:
~AutotestPrivateMouseMoveFunction() override;
ResponseAction Run() override;

void OnDone();

std::unique_ptr<EventGenerator> event_generator_;
};

template <>
KeyedService*
BrowserContextKeyedAPIFactory<AutotestPrivateAPI>::BuildServiceInstanceFor(
Expand Down
Loading

0 comments on commit 7077319

Please sign in to comment.