#include "pch.h"
#include "Direct3DInterop.h"
#include "Direct3DContentProvider.h"
#include <windows.storage.streams.h>
#include <wrl.h>
#include <robuffer.h>
#include <opencv2\core\core.hpp>
#include <opencv2\imgproc\imgproc.hpp>
#include <opencv2\features2d\features2d.hpp>
#include <algorithm>
using namespace Windows::Storage::Streams;
using namespace Microsoft::WRL;
using namespace Windows::Foundation;
using namespace Windows::UI::Core;
using namespace Microsoft::WRL;
using namespace Windows::Phone::Graphics::Interop;
using namespace Windows::Phone::Input::Interop;
using namespace Windows::Foundation;
using namespace Windows::Foundation::Collections;
using namespace Windows::Phone::Media::Capture;
#if !defined(_M_ARM)
#pragma message("warning: Direct3DInterop.cpp: Windows Phone camera code does not run in the emulator.")
#pragma message("warning: Direct3DInterop.cpp: Please compile as an ARM build and run on a device.")
#endif
namespace PhoneXamlDirect3DApp1Comp
{
// Called each time a preview frame is available
void CameraCapturePreviewSink::OnFrameAvailable(
DXGI_FORMAT format,
UINT width,
UINT height,
BYTE* pixels
)
{
m_Direct3dInterop->UpdateFrame(pixels, width, height);
}
// Called each time a captured frame is available
void CameraCaptureSampleSink::OnSampleAvailable(
ULONGLONG hnsPresentationTime,
ULONGLONG hnsSampleDuration,
DWORD cbSample,
BYTE* pSample)
{
}
Direct3DInterop::Direct3DInterop()
: m_algorithm(OCVFilterType::ePreview)
, m_contentDirty(false)
, m_backFrame(nullptr)
, m_frontFrame(nullptr)
{
}
bool Direct3DInterop::SwapFrames()
{
std::lock_guard<std::mutex> lock(m_mutex);
if(m_backFrame != nullptr)
{
std::swap(m_backFrame, m_frontFrame);
return true;
}
return false;
}
void Direct3DInterop::UpdateFrame(byte* buffer,int width,int height)
{
std::lock_guard<std::mutex> lock(m_mutex);
if(m_backFrame == nullptr)
{
m_backFrame = std::shared_ptr<cv::Mat> (new cv::Mat(height, width, CV_8UC4));
m_frontFrame = std::shared_ptr<cv::Mat> (new cv::Mat(height, width, CV_8UC4));
}
memcpy(m_backFrame.get()->data, buffer, 4 * height*width);
m_contentDirty = true;
RequestAdditionalFrame();
}
void Direct3DInterop::ProcessFrame()
{
if (SwapFrames())
{
if (m_renderer)
{
cv::Mat* mat = m_frontFrame.get();
switch (m_algorithm)
{
case OCVFilterType::ePreview:
{
break;
}
case OCVFilterType::eGray:
{
ApplyGrayFilter(mat);
break;
}
case OCVFilterType::eCanny:
{
ApplyCannyFilter(mat);
break;
}
case OCVFilterType::eBlur:
{
ApplyBlurFilter(mat);
break;
}
case OCVFilterType::eFindFeatures:
{
ApplyFindFeaturesFilter(mat);
break;
}
case OCVFilterType::eSepia:
{
ApplySepiaFilter(mat);
break;
}
}
m_renderer->CreateTextureFromByte(mat->data, mat->cols, mat->rows);
}
}
}
void Direct3DInterop::ApplyGrayFilter(cv::Mat* mat)
{
cv::Mat intermediateMat;
cv::cvtColor(*mat, intermediateMat, CV_RGBA2GRAY);
cv::cvtColor(intermediateMat, *mat, CV_GRAY2BGRA);
}
void Direct3DInterop::ApplyCannyFilter(cv::Mat* mat)
{
cv::Mat intermediateMat;
cv::Canny(*mat, intermediateMat, 80, 90);
cv::cvtColor(intermediateMat, *mat, CV_GRAY2BGRA);
}
void Direct3DInterop::ApplyBlurFilter(cv::Mat* mat)
{
cv::Mat intermediateMat;
// cv::Blur(image, intermediateMat, 80, 90);
cv::cvtColor(intermediateMat, *mat, CV_GRAY2BGRA);
}
void Direct3DInterop::ApplyFindFeaturesFilter(cv::Mat* mat)
{
cv::Mat intermediateMat;
cv::Ptr<cv::FastFeatureDetector> detector = cv::FastFeatureDetector::create(50);
std::vector<cv::KeyPoint> features;
cv::cvtColor(*mat, intermediateMat, CV_RGBA2GRAY);
detector->detect(intermediateMat, features);
for( unsigned int i = 0; i < std::min(features.size(), (size_t)50); i++ )
{
const cv::KeyPoint& kp = features[i];
cv::circle(*mat, cv::Point((int)kp.pt.x, (int)kp.pt.y), 10, cv::Scalar(255,0,0,255));
}
}
void Direct3DInterop::ApplySepiaFilter(cv::Mat* mat)
{
const float SepiaKernelData[16] =
{
/* B */0.131f, 0.534f, 0.272f, 0.f,
/* G */0.168f, 0.686f, 0.349f, 0.f,
/* R */0.189f, 0.769f, 0.393f, 0.f,
/* A */0.000f, 0.000f, 0.000f, 1.f
};
const cv::Mat SepiaKernel(4, 4, CV_32FC1, (void*)SepiaKernelData);
cv::transform(*mat, *mat, SepiaKernel);
}
IDrawingSurfaceContentProvider^ Direct3DInterop::CreateContentProvider()
{
ComPtr<Direct3DContentProvider> provider = Make<Direct3DContentProvider>(this);
return reinterpret_cast<IDrawingSurfaceContentProvider^>(provider.Detach());
}
// IDrawingSurfaceManipulationHandler
void Direct3DInterop::SetManipulationHost(DrawingSurfaceManipulationHost^ manipulationHost)
{
manipulationHost->PointerPressed +=
ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerPressed);
manipulationHost->PointerMoved +=
ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerMoved);
manipulationHost->PointerReleased +=
ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerReleased);
}
void Direct3DInterop::RenderResolution::set(Windows::Foundation::Size renderResolution)
{
if (renderResolution.Width != m_renderResolution.Width ||
renderResolution.Height != m_renderResolution.Height)
{
m_renderResolution = renderResolution;
if (m_renderer)
{
m_renderer->UpdateForRenderResolutionChange(m_renderResolution.Width, m_renderResolution.Height);
RecreateSynchronizedTexture();
}
}
}
// Event Handlers
void Direct3DInterop::OnPointerPressed(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
{
// Insert your code here.
}
void Direct3DInterop::OnPointerMoved(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
{
// Insert your code here.
}
void Direct3DInterop::OnPointerReleased(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
{
// Insert your code here.
}
void Direct3DInterop::StartCamera()
{
// Set the capture dimensions
Size captureDimensions;
captureDimensions.Width = 640;
captureDimensions.Height = 480;
// Open the AudioVideoCaptureDevice for video only
IAsyncOperation<AudioVideoCaptureDevice^> ^openOperation = AudioVideoCaptureDevice::OpenForVideoOnlyAsync(CameraSensorLocation::Back, captureDimensions);
openOperation->Completed = ref new AsyncOperationCompletedHandler<AudioVideoCaptureDevice^>(
[this] (IAsyncOperation<AudioVideoCaptureDevice^> ^operation, Windows::Foundation::AsyncStatus status)
{
if (status == Windows::Foundation::AsyncStatus::Completed)
{
auto captureDevice = operation->GetResults();
// Save the reference to the opened video capture device
pAudioVideoCaptureDevice = captureDevice;
// Retrieve the native ICameraCaptureDeviceNative interface from the managed video capture device
ICameraCaptureDeviceNative *iCameraCaptureDeviceNative = NULL;
HRESULT hr = reinterpret_cast<IUnknown*>(captureDevice)->QueryInterface(__uuidof(ICameraCaptureDeviceNative), (void**) &iCameraCaptureDeviceNative);
// Save the pointer to the native interface
pCameraCaptureDeviceNative = iCameraCaptureDeviceNative;
// Initialize the preview dimensions (see the accompanying article at )
// The aspect ratio of the capture and preview resolution must be equal,
// 4:3 for capture => 4:3 for preview, and 16:9 for capture => 16:9 for preview.
Size previewDimensions;
previewDimensions.Width = 640;
previewDimensions.Height = 480;
IAsyncAction^ setPreviewResolutionAction = pAudioVideoCaptureDevice->SetPreviewResolutionAsync(previewDimensions);
setPreviewResolutionAction->Completed = ref new AsyncActionCompletedHandler(
[this](IAsyncAction^ action, Windows::Foundation::AsyncStatus status)
{
HResult hr = action->ErrorCode;
if (status == Windows::Foundation::AsyncStatus::Completed)
{
// Create the sink
MakeAndInitialize<CameraCapturePreviewSink>(&pCameraCapturePreviewSink);
pCameraCapturePreviewSink->SetDelegate(this);
pCameraCaptureDeviceNative->SetPreviewSink(pCameraCapturePreviewSink);
// Set the preview format
pCameraCaptureDeviceNative->SetPreviewFormat(DXGI_FORMAT::DXGI_FORMAT_B8G8R8A8_UNORM);
}
}
);
// Retrieve IAudioVideoCaptureDeviceNative native interface from managed projection.
IAudioVideoCaptureDeviceNative *iAudioVideoCaptureDeviceNative = NULL;
hr = reinterpret_cast<IUnknown*>(captureDevice)->QueryInterface(__uuidof(IAudioVideoCaptureDeviceNative), (void**) &iAudioVideoCaptureDeviceNative);
// Save the pointer to the IAudioVideoCaptureDeviceNative native interface
pAudioVideoCaptureDeviceNative = iAudioVideoCaptureDeviceNative;
// Set sample encoding format to ARGB. See the documentation for further values.
pAudioVideoCaptureDevice->VideoEncodingFormat = CameraCaptureVideoFormat::Argb;
// Initialize and set the CameraCaptureSampleSink class as sink for captures samples
MakeAndInitialize<CameraCaptureSampleSink>(&pCameraCaptureSampleSink);
pAudioVideoCaptureDeviceNative->SetVideoSampleSink(pCameraCaptureSampleSink);
// Start recording (only way to receive samples using the ICameraCaptureSampleSink interface
pAudioVideoCaptureDevice->StartRecordingToSinkAsync();
}
}
);
}
// Interface With Direct3DContentProvider
HRESULT Direct3DInterop::Connect(_In_ IDrawingSurfaceRuntimeHostNative* host)
{
m_renderer = ref new QuadRenderer();
m_renderer->Initialize();
m_renderer->UpdateForWindowSizeChange(WindowBounds.Width, WindowBounds.Height);
m_renderer->UpdateForRenderResolutionChange(m_renderResolution.Width, m_renderResolution.Height);
StartCamera();
return S_OK;
}
void Direct3DInterop::Disconnect()
{
m_renderer = nullptr;
}
HRESULT Direct3DInterop::PrepareResources(_In_ const LARGE_INTEGER* presentTargetTime, _Out_ BOOL* contentDirty)
{
*contentDirty = m_contentDirty;
if(m_contentDirty)
{
ProcessFrame();
}
m_contentDirty = false;
return S_OK;
}
HRESULT Direct3DInterop::GetTexture(_In_ const DrawingSurfaceSizeF* size, _Out_ IDrawingSurfaceSynchronizedTextureNative** synchronizedTexture, _Out_ DrawingSurfaceRectF* textureSubRectangle)
{
m_renderer->Update();
m_renderer->Render();
return S_OK;
}
ID3D11Texture2D* Direct3DInterop::GetTexture()
{
return m_renderer->GetTexture();
}
}