// Copyright (c) 2013 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "win8/test/ui_automation_client.h" #include <atlbase.h> #include <atlcom.h> #include <oleauto.h> #include <uiautomation.h> #include <algorithm> #include "base/bind.h" #include "base/callback.h" #include "base/memory/ref_counted.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" #include "base/thread_task_runner_handle.h" #include "base/win/scoped_comptr.h" #include "base/win/scoped_variant.h" namespace win8 { namespace internal { // The guts of the UI automation client which runs on a dedicated thread in the // multi-threaded COM apartment. An instance may be constructed on any thread, // but Initialize() must be invoked on a thread in the MTA. class UIAutomationClient::Context { public: // Returns a new instance ready for initialization and use on another thread. static base::WeakPtr<Context> Create(); // Deletes the instance. void DeleteOnAutomationThread(); // Initializes the context, invoking |init_callback| via |client_runner| when // done. On success, |result_callback| will eventually be called after the // window has been processed. On failure, this instance self-destructs after // posting |init_callback|. void Initialize( scoped_refptr<base::SingleThreadTaskRunner> client_runner, string16 class_name, string16 item_name, UIAutomationClient::InitializedCallback init_callback, UIAutomationClient::ResultCallback result_callback); // Methods invoked by event handlers via weak pointers. void HandleAutomationEvent( base::win::ScopedComPtr<IUIAutomationElement> sender, EVENTID eventId); private: class EventHandler; // The only and only method that may be called from outside of the automation // thread. Context(); ~Context(); HRESULT InstallWindowObserver(); HRESULT RemoveWindowObserver(); void HandleWindowOpen( const base::win::ScopedComPtr<IUIAutomationElement>& window); void ProcessWindow( const base::win::ScopedComPtr<IUIAutomationElement>& window); HRESULT InvokeDesiredItem( const base::win::ScopedComPtr<IUIAutomationElement>& element); HRESULT GetInvokableItems( const base::win::ScopedComPtr<IUIAutomationElement>& element, std::vector<string16>* choices); void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window); base::ThreadChecker thread_checker_; // The loop on which the client itself lives. scoped_refptr<base::SingleThreadTaskRunner> client_runner_; // The class name of the window for which the client waits. string16 class_name_; // The name of the item to invoke. string16 item_name_; // The consumer's result callback. ResultCallback result_callback_; // The automation client. base::win::ScopedComPtr<IUIAutomation> automation_; // A handler of Window open events. base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_; // Weak pointers to the context are given to event handlers. base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_; DISALLOW_COPY_AND_ASSIGN(Context); }; class UIAutomationClient::Context::EventHandler : public CComObjectRootEx<CComMultiThreadModel>, public IUIAutomationEventHandler { public: BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler) COM_INTERFACE_ENTRY(IUIAutomationEventHandler) END_COM_MAP() EventHandler(); virtual ~EventHandler(); // Initializes the object with its parent UI automation client context's // message loop and pointer. Events are dispatched back to the context on // the given loop. void Initialize( const scoped_refptr<base::SingleThreadTaskRunner>& context_runner, const base::WeakPtr<UIAutomationClient::Context>& context); // IUIAutomationEventHandler methods. STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender, EVENTID eventId); private: // The task runner for the UI automation client context. scoped_refptr<base::SingleThreadTaskRunner> context_runner_; // The parent UI automation client context. base::WeakPtr<UIAutomationClient::Context> context_; DISALLOW_COPY_AND_ASSIGN(EventHandler); }; UIAutomationClient::Context::EventHandler::EventHandler() {} UIAutomationClient::Context::EventHandler::~EventHandler() {} void UIAutomationClient::Context::EventHandler::Initialize( const scoped_refptr<base::SingleThreadTaskRunner>& context_runner, const base::WeakPtr<UIAutomationClient::Context>& context) { context_runner_ = context_runner; context_ = context; } HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent( IUIAutomationElement* sender, EVENTID eventId) { // Event handlers are invoked on an arbitrary thread in the MTA. Send the // event back to the main UI automation thread for processing. context_runner_->PostTask( FROM_HERE, base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_, base::win::ScopedComPtr<IUIAutomationElement>(sender), eventId)); return S_OK; } base::WeakPtr<UIAutomationClient::Context> UIAutomationClient::Context::Create() { Context* context = new Context(); return context->weak_ptr_factory_.GetWeakPtr(); } void UIAutomationClient::Context::DeleteOnAutomationThread() { DCHECK(thread_checker_.CalledOnValidThread()); delete this; } UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {} UIAutomationClient::Context::~Context() { DCHECK(thread_checker_.CalledOnValidThread()); if (event_handler_.get()) { event_handler_ = NULL; HRESULT result = automation_->RemoveAllEventHandlers(); LOG_IF(ERROR, FAILED(result)) << std::hex << result; } } void UIAutomationClient::Context::Initialize( scoped_refptr<base::SingleThreadTaskRunner> client_runner, string16 class_name, string16 item_name, UIAutomationClient::InitializedCallback init_callback, UIAutomationClient::ResultCallback result_callback) { // This and all other methods must be called on the automation thread. DCHECK(!client_runner->BelongsToCurrentThread()); // Bind the checker to this thread. thread_checker_.DetachFromThread(); DCHECK(thread_checker_.CalledOnValidThread()); client_runner_ = client_runner; class_name_ = class_name; item_name_ = item_name; result_callback_ = result_callback; HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL, CLSCTX_INPROC_SERVER); if (FAILED(result) || !automation_.get()) LOG(ERROR) << std::hex << result; else result = InstallWindowObserver(); // Tell the client that initialization is complete. client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result)); // Self-destruct if the overall operation failed. if (FAILED(result)) delete this; } // Installs the window observer. HRESULT UIAutomationClient::Context::InstallWindowObserver() { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(automation_.get()); DCHECK(!event_handler_.get()); HRESULT result = S_OK; base::win::ScopedComPtr<IUIAutomationElement> root_element; base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request; // Observe the opening of all windows. result = automation_->GetRootElement(root_element.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } // Cache Window class, HWND, and window pattern for opened windows. result = automation_->CreateCacheRequest(cache_request.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } cache_request->AddProperty(UIA_ClassNamePropertyId); cache_request->AddProperty(UIA_NativeWindowHandlePropertyId); // Create the observer. CComObject<EventHandler>* event_handler_obj = NULL; result = CComObject<EventHandler>::CreateInstance(&event_handler_obj); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(), weak_ptr_factory_.GetWeakPtr()); base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler( event_handler_obj); result = automation_->AddAutomationEventHandler( UIA_Window_WindowOpenedEventId, root_element, TreeScope_Descendants, cache_request, event_handler); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } event_handler_ = event_handler; return S_OK; } // Removes this instance's window observer. HRESULT UIAutomationClient::Context::RemoveWindowObserver() { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(automation_.get()); DCHECK(event_handler_.get()); HRESULT result = S_OK; base::win::ScopedComPtr<IUIAutomationElement> root_element; // The opening of all windows are observed. result = automation_->GetRootElement(root_element.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } result = automation_->RemoveAutomationEventHandler( UIA_Window_WindowOpenedEventId, root_element, event_handler_); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } event_handler_ = NULL; return S_OK; } // Handles an automation event. If the event results in the processing for which // this context was created, the context self-destructs after posting the // results to the client. void UIAutomationClient::Context::HandleAutomationEvent( base::win::ScopedComPtr<IUIAutomationElement> sender, EVENTID eventId) { DCHECK(thread_checker_.CalledOnValidThread()); if (eventId == UIA_Window_WindowOpenedEventId) HandleWindowOpen(sender); } // Handles a WindowOpen event. If |window| is the one for which this instance is // waiting, it is processed and this instance self-destructs after posting the // results to the client. void UIAutomationClient::Context::HandleWindowOpen( const base::win::ScopedComPtr<IUIAutomationElement>& window) { DCHECK(thread_checker_.CalledOnValidThread()); HRESULT hr = S_OK; base::win::ScopedVariant var; hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE, var.Receive()); if (FAILED(hr)) { LOG(ERROR) << std::hex << hr; return; } if (V_VT(&var) != VT_BSTR) { LOG(ERROR) << __FUNCTION__ " class name is not a BSTR: " << V_VT(&var); return; } string16 class_name(V_BSTR(&var)); // Window class names are atoms, which are case-insensitive. if (class_name.size() == class_name_.size() && std::equal(class_name.begin(), class_name.end(), class_name_.begin(), base::CaseInsensitiveCompare<wchar_t>())) { RemoveWindowObserver(); ProcessWindow(window); } } // Processes |window| by invoking the desired child item. If the item cannot be // found or invoked, an attempt is made to get a list of all invokable children. // The results are posted back to the client on |client_runner_|, and this // instance self-destructs. void UIAutomationClient::Context::ProcessWindow( const base::win::ScopedComPtr<IUIAutomationElement>& window) { DCHECK(thread_checker_.CalledOnValidThread()); HRESULT result = S_OK; std::vector<string16> choices; result = InvokeDesiredItem(window); if (FAILED(result)) { GetInvokableItems(window, &choices); CloseWindow(window); } client_runner_->PostTask(FROM_HERE, base::Bind(result_callback_, result, choices)); // Self-destruct since there's nothing more to be done here. delete this; } // Invokes the desired child of |element|. HRESULT UIAutomationClient::Context::InvokeDesiredItem( const base::win::ScopedComPtr<IUIAutomationElement>& element) { DCHECK(thread_checker_.CalledOnValidThread()); HRESULT result = S_OK; base::win::ScopedVariant var; base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition; base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition; base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition; base::win::ScopedComPtr<IUIAutomationCondition> condition; base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request; base::win::ScopedComPtr<IUIAutomationElement> target; // Search for an invokable element named item_name. var.Set(true); result = automation_->CreatePropertyCondition( UIA_IsInvokePatternAvailablePropertyId, var, invokable_condition.Receive()); var.Reset(); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return false; } var.Set(item_name_.c_str()); result = automation_->CreatePropertyCondition(UIA_NamePropertyId, var, item_name_condition.Receive()); var.Reset(); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } result = automation_->get_ControlViewCondition( control_view_condition.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } std::vector<IUIAutomationCondition*> conditions; conditions.push_back(invokable_condition.get()); conditions.push_back(item_name_condition.get()); conditions.push_back(control_view_condition.get()); result = automation_->CreateAndConditionFromNativeArray( &conditions[0], conditions.size(), condition.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } // Cache invokable pattern for the item. result = automation_->CreateCacheRequest(cache_request.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } cache_request->AddPattern(UIA_InvokePatternId); result = element->FindFirstBuildCache( static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants), condition, cache_request, target.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } // If the item was found, invoke it. if (!target.get()) { LOG(WARNING) << "Failed to find desired item to invoke."; return E_FAIL; } base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker; result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(), invoker.ReceiveVoid()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } result = invoker->Invoke(); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } return S_OK; } // Populates |choices| with the names of all invokable children of |element|. HRESULT UIAutomationClient::Context::GetInvokableItems( const base::win::ScopedComPtr<IUIAutomationElement>& element, std::vector<string16>* choices) { DCHECK(choices); DCHECK(thread_checker_.CalledOnValidThread()); HRESULT result = S_OK; base::win::ScopedVariant var; base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition; base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition; base::win::ScopedComPtr<IUIAutomationCondition> condition; base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request; base::win::ScopedComPtr<IUIAutomationElementArray> element_array; base::win::ScopedComPtr<IUIAutomationElement> child_element; // Search for all invokable elements. var.Set(true); result = automation_->CreatePropertyCondition( UIA_IsInvokePatternAvailablePropertyId, var, invokable_condition.Receive()); var.Reset(); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } result = automation_->get_ControlViewCondition( control_view_condition.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } result = automation_->CreateAndCondition( invokable_condition, control_view_condition, condition.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } // Cache item names. result = automation_->CreateCacheRequest(cache_request.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } cache_request->AddProperty(UIA_NamePropertyId); result = element->FindAllBuildCache( static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants), condition, cache_request, element_array.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } if (!element_array.get()) { LOG(ERROR) << "The window may have vanished."; return S_OK; } int num_elements = 0; result = element_array->get_Length(&num_elements); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return result; } choices->clear(); choices->reserve(num_elements); for (int i = 0; i < num_elements; ++i) { child_element.Release(); result = element_array->GetElement(i, child_element.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; continue; } result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE, var.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; continue; } if (V_VT(&var) != VT_BSTR) { LOG(ERROR) << __FUNCTION__ " name is not a BSTR: " << V_VT(&var); continue; } choices->push_back(string16(V_BSTR(&var))); var.Reset(); } return result; } // Closes the element |window| by sending it an escape key. void UIAutomationClient::Context::CloseWindow( const base::win::ScopedComPtr<IUIAutomationElement>& window) { DCHECK(thread_checker_.CalledOnValidThread()); // It's tempting to get the Window pattern from |window| and invoke its Close // method. Unfortunately, this doesn't work. Sending an escape key does the // trick, though. HRESULT result = S_OK; base::win::ScopedVariant var; result = window->GetCachedPropertyValueEx( UIA_NativeWindowHandlePropertyId, TRUE, var.Receive()); if (FAILED(result)) { LOG(ERROR) << std::hex << result; return; } if (V_VT(&var) != VT_I4) { LOG(ERROR) << __FUNCTION__ " window handle is not an int: " << V_VT(&var); return; } HWND handle = reinterpret_cast<HWND>(V_I4(&var)); uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC); PostMessage(handle, WM_KEYDOWN, VK_ESCAPE, MAKELPARAM(1, scan_code)); PostMessage(handle, WM_KEYUP, VK_ESCAPE, MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP)); } UIAutomationClient::UIAutomationClient() : automation_thread_("UIAutomation") {} UIAutomationClient::~UIAutomationClient() { DCHECK(thread_checker_.CalledOnValidThread()); // context_ is still valid when the caller destroys the instance before the // callback(s) have fired. In this case, delete the context on the automation // thread before joining with it. automation_thread_.message_loop()->PostTask( FROM_HERE, base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread, context_)); } void UIAutomationClient::Begin(const wchar_t* class_name, const string16& item_name, const InitializedCallback& init_callback, const ResultCallback& result_callback) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK_EQ(context_.get(), static_cast<Context*>(NULL)); // Start the automation thread and initialize our automation client on it. context_ = Context::Create(); automation_thread_.init_com_with_mta(true); automation_thread_.Start(); automation_thread_.message_loop()->PostTask( FROM_HERE, base::Bind(&UIAutomationClient::Context::Initialize, context_, base::ThreadTaskRunnerHandle::Get(), string16(class_name), item_name, init_callback, result_callback)); } } // namespace internal } // namespace win8