/*
* Copyright © 2013-2015 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "isl/isl.h"
#include "brw_fs_surface_builder.h"
#include "brw_fs.h"
using namespace brw;
namespace brw {
namespace surface_access {
namespace {
/**
* Generate a logical send opcode for a surface message and return
* the result.
*/
fs_reg
emit_send(const fs_builder &bld, enum opcode opcode,
const fs_reg &addr, const fs_reg &src, const fs_reg &surface,
unsigned dims, unsigned arg, unsigned rsize,
brw_predicate pred = BRW_PREDICATE_NONE)
{
/* Reduce the dynamically uniform surface index to a single
* scalar.
*/
const fs_reg usurface = bld.emit_uniformize(surface);
const fs_reg srcs[] = {
addr, src, usurface, brw_imm_ud(dims), brw_imm_ud(arg)
};
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, rsize);
fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));
inst->size_written = rsize * dst.component_size(inst->exec_size);
inst->predicate = pred;
return dst;
}
}
/**
* Emit an untyped surface read opcode. \p dims determines the number
* of components of the address and \p size the number of components of
* the returned value.
*/
fs_reg
emit_untyped_read(const fs_builder &bld,
const fs_reg &surface, const fs_reg &addr,
unsigned dims, unsigned size,
brw_predicate pred)
{
return emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
addr, fs_reg(), surface, dims, size, size, pred);
}
/**
* Emit an untyped surface write opcode. \p dims determines the number
* of components of the address and \p size the number of components of
* the argument.
*/
void
emit_untyped_write(const fs_builder &bld, const fs_reg &surface,
const fs_reg &addr, const fs_reg &src,
unsigned dims, unsigned size,
brw_predicate pred)
{
emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
addr, src, surface, dims, size, 0, pred);
}
/**
* Emit an untyped surface atomic opcode. \p dims determines the number
* of components of the address and \p rsize the number of components of
* the returned value (either zero or one).
*/
fs_reg
emit_untyped_atomic(const fs_builder &bld,
const fs_reg &surface, const fs_reg &addr,
const fs_reg &src0, const fs_reg &src1,
unsigned dims, unsigned rsize, unsigned op,
brw_predicate pred)
{
/* FINISHME: Factor out this frequently recurring pattern into a
* helper function.
*/
const unsigned n = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
const fs_reg srcs[] = { src0, src1 };
const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, n);
bld.LOAD_PAYLOAD(tmp, srcs, n, 0);
return emit_send(bld, SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
addr, tmp, surface, dims, op, rsize, pred);
}
/**
* Emit a typed surface read opcode. \p dims determines the number of
* components of the address and \p size the number of components of the
* returned value.
*/
fs_reg
emit_typed_read(const fs_builder &bld, const fs_reg &surface,
const fs_reg &addr, unsigned dims, unsigned size)
{
return emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL,
addr, fs_reg(), surface, dims, size, size);
}
/**
* Emit a typed surface write opcode. \p dims determines the number of
* components of the address and \p size the number of components of the
* argument.
*/
void
emit_typed_write(const fs_builder &bld, const fs_reg &surface,
const fs_reg &addr, const fs_reg &src,
unsigned dims, unsigned size)
{
emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL,
addr, src, surface, dims, size, 0);
}
/**
* Emit a typed surface atomic opcode. \p dims determines the number of
* components of the address and \p rsize the number of components of
* the returned value (either zero or one).
*/
fs_reg
emit_typed_atomic(const fs_builder &bld, const fs_reg &surface,
const fs_reg &addr,
const fs_reg &src0, const fs_reg &src1,
unsigned dims, unsigned rsize, unsigned op,
brw_predicate pred)
{
/* FINISHME: Factor out this frequently recurring pattern into a
* helper function.
*/
const unsigned n = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
const fs_reg srcs[] = { src0, src1 };
const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, n);
bld.LOAD_PAYLOAD(tmp, srcs, n, 0);
return emit_send(bld, SHADER_OPCODE_TYPED_ATOMIC_LOGICAL,
addr, tmp, surface, dims, op, rsize);
}
fs_reg
emit_byte_scattered_read(const fs_builder &bld,
const fs_reg &surface, const fs_reg &addr,
unsigned dims, unsigned size,
unsigned bit_size, brw_predicate pred)
{
return emit_send(bld, SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
addr, fs_reg(), surface, dims, bit_size, size, pred);
}
void
emit_byte_scattered_write(const fs_builder &bld, const fs_reg &surface,
const fs_reg &addr, const fs_reg &src,
unsigned dims, unsigned size,
unsigned bit_size, brw_predicate pred)
{
emit_send(bld, SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
addr, src, surface, dims, bit_size, 0, pred);
}
}
}
namespace {
namespace image_format_info {
/* The higher compiler layers use the GL enums for image formats even if
* they come in from SPIR-V or Vulkan. We need to turn them into an ISL
* enum before we can use them.
*/
static enum isl_format
isl_format_for_gl_format(uint32_t gl_format)
{
switch (gl_format) {
case GL_R8: return ISL_FORMAT_R8_UNORM;
case GL_R8_SNORM: return ISL_FORMAT_R8_SNORM;
case GL_R8UI: return ISL_FORMAT_R8_UINT;
case GL_R8I: return ISL_FORMAT_R8_SINT;
case GL_RG8: return ISL_FORMAT_R8G8_UNORM;
case GL_RG8_SNORM: return ISL_FORMAT_R8G8_SNORM;
case GL_RG8UI: return ISL_FORMAT_R8G8_UINT;
case GL_RG8I: return ISL_FORMAT_R8G8_SINT;
case GL_RGBA8: return ISL_FORMAT_R8G8B8A8_UNORM;
case GL_RGBA8_SNORM: return ISL_FORMAT_R8G8B8A8_SNORM;
case GL_RGBA8UI: return ISL_FORMAT_R8G8B8A8_UINT;
case GL_RGBA8I: return ISL_FORMAT_R8G8B8A8_SINT;
case GL_R11F_G11F_B10F: return ISL_FORMAT_R11G11B10_FLOAT;
case GL_RGB10_A2: return ISL_FORMAT_R10G10B10A2_UNORM;
case GL_RGB10_A2UI: return ISL_FORMAT_R10G10B10A2_UINT;
case GL_R16: return ISL_FORMAT_R16_UNORM;
case GL_R16_SNORM: return ISL_FORMAT_R16_SNORM;
case GL_R16F: return ISL_FORMAT_R16_FLOAT;
case GL_R16UI: return ISL_FORMAT_R16_UINT;
case GL_R16I: return ISL_FORMAT_R16_SINT;
case GL_RG16: return ISL_FORMAT_R16G16_UNORM;
case GL_RG16_SNORM: return ISL_FORMAT_R16G16_SNORM;
case GL_RG16F: return ISL_FORMAT_R16G16_FLOAT;
case GL_RG16UI: return ISL_FORMAT_R16G16_UINT;
case GL_RG16I: return ISL_FORMAT_R16G16_SINT;
case GL_RGBA16: return ISL_FORMAT_R16G16B16A16_UNORM;
case GL_RGBA16_SNORM: return ISL_FORMAT_R16G16B16A16_SNORM;
case GL_RGBA16F: return ISL_FORMAT_R16G16B16A16_FLOAT;
case GL_RGBA16UI: return ISL_FORMAT_R16G16B16A16_UINT;
case GL_RGBA16I: return ISL_FORMAT_R16G16B16A16_SINT;
case GL_R32F: return ISL_FORMAT_R32_FLOAT;
case GL_R32UI: return ISL_FORMAT_R32_UINT;
case GL_R32I: return ISL_FORMAT_R32_SINT;
case GL_RG32F: return ISL_FORMAT_R32G32_FLOAT;
case GL_RG32UI: return ISL_FORMAT_R32G32_UINT;
case GL_RG32I: return ISL_FORMAT_R32G32_SINT;
case GL_RGBA32F: return ISL_FORMAT_R32G32B32A32_FLOAT;
case GL_RGBA32UI: return ISL_FORMAT_R32G32B32A32_UINT;
case GL_RGBA32I: return ISL_FORMAT_R32G32B32A32_SINT;
case GL_NONE: return ISL_FORMAT_UNSUPPORTED;
default:
assert(!"Invalid image format");
return ISL_FORMAT_UNSUPPORTED;
}
}
/**
* Simple 4-tuple of scalars used to pass around per-color component
* values.
*/
struct color_u {
color_u(unsigned x = 0) : r(x), g(x), b(x), a(x)
{
}
color_u(unsigned r, unsigned g, unsigned b, unsigned a) :
r(r), g(g), b(b), a(a)
{
}
unsigned
operator[](unsigned i) const
{
const unsigned xs[] = { r, g, b, a };
return xs[i];
}
unsigned r, g, b, a;
};
/**
* Return the per-channel bitfield widths for a given image format.
*/
inline color_u
get_bit_widths(isl_format format)
{
const isl_format_layout *fmtl = isl_format_get_layout(format);
return color_u(fmtl->channels.r.bits,
fmtl->channels.g.bits,
fmtl->channels.b.bits,
fmtl->channels.a.bits);
}
/**
* Return the per-channel bitfield shifts for a given image format.
*/
inline color_u
get_bit_shifts(isl_format format)
{
const color_u widths = get_bit_widths(format);
return color_u(0, widths.r, widths.r + widths.g,
widths.r + widths.g + widths.b);
}
/**
* Return true if all present components have the same bit width.
*/
inline bool
is_homogeneous(isl_format format)
{
const color_u widths = get_bit_widths(format);
return ((widths.g == 0 || widths.g == widths.r) &&
(widths.b == 0 || widths.b == widths.r) &&
(widths.a == 0 || widths.a == widths.r));
}
/**
* Return true if the format conversion boils down to a trivial copy.
*/
inline bool
is_conversion_trivial(const gen_device_info *devinfo, isl_format format)
{
return (get_bit_widths(format).r == 32 && is_homogeneous(format)) ||
format == isl_lower_storage_image_format(devinfo, format);
}
/**
* Return true if the hardware natively supports some format with
* compatible bitfield layout, but possibly different data types.
*/
inline bool
has_supported_bit_layout(const gen_device_info *devinfo,
isl_format format)
{
const color_u widths = get_bit_widths(format);
const color_u lower_widths = get_bit_widths(
isl_lower_storage_image_format(devinfo, format));
return (widths.r == lower_widths.r &&
widths.g == lower_widths.g &&
widths.b == lower_widths.b &&
widths.a == lower_widths.a);
}
/**
* Return true if we are required to spread individual components over
* several components of the format used by the hardware (RG32 and
* friends implemented as RGBA16UI).
*/
inline bool
has_split_bit_layout(const gen_device_info *devinfo, isl_format format)
{
const isl_format lower_format =
isl_lower_storage_image_format(devinfo, format);
return (isl_format_get_num_channels(format) <
isl_format_get_num_channels(lower_format));
}
/**
* Return true if the hardware returns garbage in the unused high bits
* of each component. This may happen on IVB because we rely on the
* undocumented behavior that typed reads from surfaces of the
* unsupported R8 and R16 formats return useful data in their least
* significant bits.
*/
inline bool
has_undefined_high_bits(const gen_device_info *devinfo,
isl_format format)
{
const isl_format lower_format =
isl_lower_storage_image_format(devinfo, format);
return (devinfo->gen == 7 && !devinfo->is_haswell &&
(lower_format == ISL_FORMAT_R16_UINT ||
lower_format == ISL_FORMAT_R8_UINT));
}
/**
* Return true if the format represents values as signed integers
* requiring sign extension when unpacking.
*/
inline bool
needs_sign_extension(isl_format format)
{
return isl_format_has_snorm_channel(format) ||
isl_format_has_sint_channel(format);
}
}
namespace image_validity {
/**
* Check whether the bound image is suitable for untyped access.
*/
static brw_predicate
emit_untyped_image_check(const fs_builder &bld, const fs_reg &image,
brw_predicate pred)
{
const gen_device_info *devinfo = bld.shader->devinfo;
const fs_reg stride = offset(image, bld, BRW_IMAGE_PARAM_STRIDE_OFFSET);
if (devinfo->gen == 7 && !devinfo->is_haswell) {
/* Check whether the first stride component (i.e. the Bpp value)
* is greater than four, what on Gen7 indicates that a surface of
* type RAW has been bound for untyped access. Reading or writing
* to a surface of type other than RAW using untyped surface
* messages causes a hang on IVB and VLV.
*/
set_predicate(pred,
bld.CMP(bld.null_reg_ud(), stride, brw_imm_d(4),
BRW_CONDITIONAL_G));
return BRW_PREDICATE_NORMAL;
} else {
/* More recent generations handle the format mismatch
* gracefully.
*/
return pred;
}
}
/**
* Check whether there is an image bound at the given index and write
* the comparison result to f0.0. Returns an appropriate predication
* mode to use on subsequent image operations.
*/
static brw_predicate
emit_typed_atomic_check(const fs_builder &bld, const fs_reg &image)
{
const gen_device_info *devinfo = bld.shader->devinfo;
const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);
if (devinfo->gen == 7 && !devinfo->is_haswell) {
/* Check the first component of the size field to find out if the
* image is bound. Necessary on IVB for typed atomics because
* they don't seem to respect null surfaces and will happily
* corrupt or read random memory when no image is bound.
*/
bld.CMP(bld.null_reg_ud(),
retype(size, BRW_REGISTER_TYPE_UD),
brw_imm_d(0), BRW_CONDITIONAL_NZ);
return BRW_PREDICATE_NORMAL;
} else {
/* More recent platforms implement compliant behavior when a null
* surface is bound.
*/
return BRW_PREDICATE_NONE;
}
}
/**
* Check whether the provided coordinates are within the image bounds
* and write the comparison result to f0.0. Returns an appropriate
* predication mode to use on subsequent image operations.
*/
static brw_predicate
emit_bounds_check(const fs_builder &bld, const fs_reg &image,
const fs_reg &addr, unsigned dims)
{
const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);
for (unsigned c = 0; c < dims; ++c)
set_predicate(c == 0 ? BRW_PREDICATE_NONE : BRW_PREDICATE_NORMAL,
bld.CMP(bld.null_reg_ud(),
offset(retype(addr, BRW_REGISTER_TYPE_UD), bld, c),
offset(size, bld, c),
BRW_CONDITIONAL_L));
return BRW_PREDICATE_NORMAL;
}
}
namespace image_coordinates {
/**
* Return the total number of coordinates needed to address a texel of
* the surface, which may be more than the sum of \p surf_dims and \p
* arr_dims if padding is required.
*/
static unsigned
num_image_coordinates(const fs_builder &bld,
unsigned surf_dims, unsigned arr_dims,
isl_format format)
{
/* HSW in vec4 mode and our software coordinate handling for untyped
* reads want the array index to be at the Z component.
*/
const bool array_index_at_z =
format != ISL_FORMAT_UNSUPPORTED &&
!isl_has_matching_typed_storage_image_format(
bld.shader->devinfo, format);
const unsigned zero_dims =
((surf_dims == 1 && arr_dims == 1 && array_index_at_z) ? 1 : 0);
return surf_dims + zero_dims + arr_dims;
}
/**
* Transform image coordinates into the form expected by the
* implementation.
*/
static fs_reg
emit_image_coordinates(const fs_builder &bld, const fs_reg &addr,
unsigned surf_dims, unsigned arr_dims,
isl_format format)
{
const unsigned dims =
num_image_coordinates(bld, surf_dims, arr_dims, format);
if (dims > surf_dims + arr_dims) {
assert(surf_dims == 1 && arr_dims == 1 && dims == 3);
/* The array index is required to be passed in as the Z component,
* insert a zero at the Y component to shift it to the right
* position.
*
* FINISHME: Factor out this frequently recurring pattern into a
* helper function.
*/
const fs_reg srcs[] = { addr, brw_imm_d(0), offset(addr, bld, 1) };
const fs_reg dst = bld.vgrf(addr.type, dims);
bld.LOAD_PAYLOAD(dst, srcs, dims, 0);
return dst;
} else {
return addr;
}
}
/**
* Calculate the offset in memory of the texel given by \p coord.
*
* This is meant to be used with untyped surface messages to access a
* tiled surface, what involves taking into account the tiling and
* swizzling modes of the surface manually so it will hopefully not
* happen very often.
*
* The tiling algorithm implemented here matches either the X or Y
* tiling layouts supported by the hardware depending on the tiling
* coefficients passed to the program as uniforms. See Volume 1 Part 2
* Section 4.5 "Address Tiling Function" of the IVB PRM for an in-depth
* explanation of the hardware tiling format.
*/
static fs_reg
emit_address_calculation(const fs_builder &bld, const fs_reg &image,
const fs_reg &coord, unsigned dims)
{
const gen_device_info *devinfo = bld.shader->devinfo;
const fs_reg off = offset(image, bld, BRW_IMAGE_PARAM_OFFSET_OFFSET);
const fs_reg stride = offset(image, bld, BRW_IMAGE_PARAM_STRIDE_OFFSET);
const fs_reg tile = offset(image, bld, BRW_IMAGE_PARAM_TILING_OFFSET);
const fs_reg swz = offset(image, bld, BRW_IMAGE_PARAM_SWIZZLING_OFFSET);
const fs_reg addr = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
const fs_reg minor = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
const fs_reg major = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD);
/* Shift the coordinates by the fixed surface offset. It may be
* non-zero if the image is a single slice of a higher-dimensional
* surface, or if a non-zero mipmap level of the surface is bound to
* the pipeline. The offset needs to be applied here rather than at
* surface state set-up time because the desired slice-level may
* start mid-tile, so simply shifting the surface base address
* wouldn't give a well-formed tiled surface in the general case.
*/
for (unsigned c = 0; c < 2; ++c)
bld.ADD(offset(addr, bld, c), offset(off, bld, c),
(c < dims ?
offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, c) :
fs_reg(brw_imm_d(0))));
/* The layout of 3-D textures in memory is sort-of like a tiling
* format. At each miplevel, the slices are arranged in rows of
* 2^level slices per row. The slice row is stored in tmp.y and
* the slice within the row is stored in tmp.x.
*
* The layout of 2-D array textures and cubemaps is much simpler:
* Depending on whether the ARYSPC_LOD0 layout is in use it will be
* stored in memory as an array of slices, each one being a 2-D
* arrangement of miplevels, or as a 2D arrangement of miplevels,
* each one being an array of slices. In either case the separation
* between slices of the same LOD is equal to the qpitch value
* provided as stride.w.
*
* This code can be made to handle either 2D arrays and 3D textures
* by passing in the miplevel as tile.z for 3-D textures and 0 in
* tile.z for 2-D array textures.
*
* See Volume 1 Part 1 of the Gen7 PRM, sections 6.18.4.7 "Surface
* Arrays" and 6.18.6 "3D Surfaces" for a more extensive discussion
* of the hardware 3D texture and 2D array layouts.
*/
if (dims > 2) {
/* Decompose z into a major (tmp.y) and a minor (tmp.x)
* index.
*/
bld.BFE(offset(tmp, bld, 0), offset(tile, bld, 2), brw_imm_d(0),
offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, 2));
bld.SHR(offset(tmp, bld, 1),
offset(retype(coord, BRW_REGISTER_TYPE_UD), bld, 2),
offset(tile, bld, 2));
/* Take into account the horizontal (tmp.x) and vertical (tmp.y)
* slice offset.
*/
for (unsigned c = 0; c < 2; ++c) {
bld.MUL(offset(tmp, bld, c),
offset(stride, bld, 2 + c), offset(tmp, bld, c));
bld.ADD(offset(addr, bld, c),
offset(addr, bld, c), offset(tmp, bld, c));
}
}
if (dims > 1) {
/* Calculate the major/minor x and y indices. In order to
* accommodate both X and Y tiling, the Y-major tiling format is
* treated as being a bunch of narrow X-tiles placed next to each
* other. This means that the tile width for Y-tiling is actually
* the width of one sub-column of the Y-major tile where each 4K
* tile has 8 512B sub-columns.
*
* The major Y value is the row of tiles in which the pixel lives.
* The major X value is the tile sub-column in which the pixel
* lives; for X tiling, this is the same as the tile column, for Y
* tiling, each tile has 8 sub-columns. The minor X and Y indices
* are the position within the sub-column.
*/
for (unsigned c = 0; c < 2; ++c) {
/* Calculate the minor x and y indices. */
bld.BFE(offset(minor, bld, c), offset(tile, bld, c),
brw_imm_d(0), offset(addr, bld, c));
/* Calculate the major x and y indices. */
bld.SHR(offset(major, bld, c),
offset(addr, bld, c), offset(tile, bld, c));
}
/* Calculate the texel index from the start of the tile row and
* the vertical coordinate of the row.
* Equivalent to:
* tmp.x = (major.x << tile.y << tile.x) +
* (minor.y << tile.x) + minor.x
* tmp.y = major.y << tile.y
*/
bld.SHL(tmp, major, offset(tile, bld, 1));
bld.ADD(tmp, tmp, offset(minor, bld, 1));
bld.SHL(tmp, tmp, offset(tile, bld, 0));
bld.ADD(tmp, tmp, minor);
bld.SHL(offset(tmp, bld, 1),
offset(major, bld, 1), offset(tile, bld, 1));
/* Add it to the start of the tile row. */
bld.MUL(offset(tmp, bld, 1),
offset(tmp, bld, 1), offset(stride, bld, 1));
bld.ADD(tmp, tmp, offset(tmp, bld, 1));
/* Multiply by the Bpp value. */
bld.MUL(dst, tmp, stride);
if (devinfo->gen < 8 && !devinfo->is_baytrail) {
/* Take into account the two dynamically specified shifts.
* Both need are used to implement swizzling of X-tiled
* surfaces. For Y-tiled surfaces only one bit needs to be
* XOR-ed with bit 6 of the memory address, so a swz value of
* 0xff (actually interpreted as 31 by the hardware) will be
* provided to cause the relevant bit of tmp.y to be zero and
* turn the first XOR into the identity. For linear surfaces
* or platforms lacking address swizzling both shifts will be
* 0xff causing the relevant bits of both tmp.x and .y to be
* zero, what effectively disables swizzling.
*/
for (unsigned c = 0; c < 2; ++c)
bld.SHR(offset(tmp, bld, c), dst, offset(swz, bld, c));
/* XOR tmp.x and tmp.y with bit 6 of the memory address. */
bld.XOR(tmp, tmp, offset(tmp, bld, 1));
bld.AND(tmp, tmp, brw_imm_d(1 << 6));
bld.XOR(dst, dst, tmp);
}
} else {
/* Multiply by the Bpp/stride value. Note that the addr.y may be
* non-zero even if the image is one-dimensional because a
* vertical offset may have been applied above to select a
* non-zero slice or level of a higher-dimensional texture.
*/
bld.MUL(offset(addr, bld, 1),
offset(addr, bld, 1), offset(stride, bld, 1));
bld.ADD(addr, addr, offset(addr, bld, 1));
bld.MUL(dst, addr, stride);
}
return dst;
}
}
namespace image_format_conversion {
using image_format_info::color_u;
namespace {
/**
* Maximum representable value in an unsigned integer with the given
* number of bits.
*/
inline unsigned
scale(unsigned n)
{
return (1 << n) - 1;
}
}
/**
* Pack the vector \p src in a bitfield given the per-component bit
* shifts and widths. Note that bitfield components are not allowed to
* cross 32-bit boundaries.
*/
static fs_reg
emit_pack(const fs_builder &bld, const fs_reg &src,
const color_u &shifts, const color_u &widths)
{
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
bool seen[4] = {};
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
/* Shift each component left to the correct bitfield position. */
bld.SHL(tmp, offset(src, bld, c), brw_imm_ud(shifts[c] % 32));
/* Add everything up. */
if (seen[shifts[c] / 32]) {
bld.OR(offset(dst, bld, shifts[c] / 32),
offset(dst, bld, shifts[c] / 32), tmp);
} else {
bld.MOV(offset(dst, bld, shifts[c] / 32), tmp);
seen[shifts[c] / 32] = true;
}
}
}
return dst;
}
/**
* Unpack a vector from the bitfield \p src given the per-component bit
* shifts and widths. Note that bitfield components are not allowed to
* cross 32-bit boundaries.
*/
static fs_reg
emit_unpack(const fs_builder &bld, const fs_reg &src,
const color_u &shifts, const color_u &widths)
{
const fs_reg dst = bld.vgrf(src.type, 4);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
/* Shift left to discard the most significant bits. */
bld.SHL(offset(dst, bld, c),
offset(src, bld, shifts[c] / 32),
brw_imm_ud(32 - shifts[c] % 32 - widths[c]));
/* Shift back to the least significant bits using an arithmetic
* shift to get sign extension on signed types.
*/
bld.ASR(offset(dst, bld, c),
offset(dst, bld, c), brw_imm_ud(32 - widths[c]));
}
}
return dst;
}
/**
* Convert an integer vector into another integer vector of the
* specified bit widths, properly handling overflow.
*/
static fs_reg
emit_convert_to_integer(const fs_builder &bld, const fs_reg &src,
const color_u &widths, bool is_signed)
{
const unsigned s = (is_signed ? 1 : 0);
const fs_reg dst = bld.vgrf(
is_signed ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_UD, 4);
assert(src.type == dst.type);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
/* Clamp to the maximum value. */
bld.emit_minmax(offset(dst, bld, c), offset(src, bld, c),
brw_imm_d((int)scale(widths[c] - s)),
BRW_CONDITIONAL_L);
/* Clamp to the minimum value. */
if (is_signed)
bld.emit_minmax(offset(dst, bld, c), offset(dst, bld, c),
brw_imm_d(-(int)scale(widths[c] - s) - 1),
BRW_CONDITIONAL_GE);
/* Mask off all but the bits we actually want. Otherwise, if
* we pass a negative number into the hardware when it's
* expecting something like UINT8, it will happily clamp it to
* +255 for us.
*/
if (is_signed && widths[c] < 32)
bld.AND(offset(dst, bld, c), offset(dst, bld, c),
brw_imm_d(scale(widths[c])));
}
}
return dst;
}
/**
* Convert a normalized fixed-point vector of the specified signedness
* and bit widths into a floating point vector.
*/
static fs_reg
emit_convert_from_scaled(const fs_builder &bld, const fs_reg &src,
const color_u &widths, bool is_signed)
{
const unsigned s = (is_signed ? 1 : 0);
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
/* Convert to float. */
bld.MOV(offset(dst, bld, c), offset(src, bld, c));
/* Divide by the normalization constants. */
bld.MUL(offset(dst, bld, c), offset(dst, bld, c),
brw_imm_f(1.0f / scale(widths[c] - s)));
/* Clamp to the minimum value. */
if (is_signed)
bld.emit_minmax(offset(dst, bld, c),
offset(dst, bld, c), brw_imm_f(-1.0f),
BRW_CONDITIONAL_GE);
}
}
return dst;
}
/**
* Convert a floating-point vector into a normalized fixed-point vector
* of the specified signedness and bit widths.
*/
static fs_reg
emit_convert_to_scaled(const fs_builder &bld, const fs_reg &src,
const color_u &widths, bool is_signed)
{
const unsigned s = (is_signed ? 1 : 0);
const fs_reg dst = bld.vgrf(
is_signed ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_UD, 4);
const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
/* Clamp the normalized floating-point argument. */
if (is_signed) {
bld.emit_minmax(offset(fdst, bld, c), offset(src, bld, c),
brw_imm_f(-1.0f), BRW_CONDITIONAL_GE);
bld.emit_minmax(offset(fdst, bld, c), offset(fdst, bld, c),
brw_imm_f(1.0f), BRW_CONDITIONAL_L);
} else {
set_saturate(true, bld.MOV(offset(fdst, bld, c),
offset(src, bld, c)));
}
/* Multiply by the normalization constants. */
bld.MUL(offset(fdst, bld, c), offset(fdst, bld, c),
brw_imm_f((float)scale(widths[c] - s)));
/* Convert to integer. */
bld.RNDE(offset(fdst, bld, c), offset(fdst, bld, c));
bld.MOV(offset(dst, bld, c), offset(fdst, bld, c));
/* Mask off all but the bits we actually want. Otherwise, if
* we pass a negative number into the hardware when it's
* expecting something like UINT8, it will happily clamp it to
* +255 for us.
*/
if (is_signed && widths[c] < 32)
bld.AND(offset(dst, bld, c), offset(dst, bld, c),
brw_imm_d(scale(widths[c])));
}
}
return dst;
}
/**
* Convert a floating point vector of the specified bit widths into a
* 32-bit floating point vector.
*/
static fs_reg
emit_convert_from_float(const fs_builder &bld, const fs_reg &src,
const color_u &widths)
{
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
bld.MOV(offset(dst, bld, c), offset(src, bld, c));
/* Extend 10-bit and 11-bit floating point numbers to 15 bits.
* This works because they have a 5-bit exponent just like the
* 16-bit floating point format, and they have no sign bit.
*/
if (widths[c] < 16)
bld.SHL(offset(dst, bld, c),
offset(dst, bld, c), brw_imm_ud(15 - widths[c]));
/* Convert to 32-bit floating point. */
bld.F16TO32(offset(fdst, bld, c), offset(dst, bld, c));
}
}
return fdst;
}
/**
* Convert a vector into a floating point vector of the specified bit
* widths.
*/
static fs_reg
emit_convert_to_float(const fs_builder &bld, const fs_reg &src,
const color_u &widths)
{
const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
const fs_reg fdst = retype(dst, BRW_REGISTER_TYPE_F);
for (unsigned c = 0; c < 4; ++c) {
if (widths[c]) {
bld.MOV(offset(fdst, bld, c), offset(src, bld, c));
/* Clamp to the minimum value. */
if (widths[c] < 16)
bld.emit_minmax(offset(fdst, bld, c), offset(fdst, bld, c),
brw_imm_f(0.0f), BRW_CONDITIONAL_GE);
/* Convert to 16-bit floating-point. */
bld.F32TO16(offset(dst, bld, c), offset(fdst, bld, c));
/* Discard the least significant bits to get floating point
* numbers of the requested width. This works because the
* 10-bit and 11-bit floating point formats have a 5-bit
* exponent just like the 16-bit format, and they have no sign
* bit.
*/
if (widths[c] < 16)
bld.SHR(offset(dst, bld, c), offset(dst, bld, c),
brw_imm_ud(15 - widths[c]));
}
}
return dst;
}
/**
* Fill missing components of a vector with 0, 0, 0, 1.
*/
static fs_reg
emit_pad(const fs_builder &bld, const fs_reg &src,
const color_u &widths)
{
const fs_reg dst = bld.vgrf(src.type, 4);
const unsigned pad[] = { 0, 0, 0, 1 };
for (unsigned c = 0; c < 4; ++c)
bld.MOV(offset(dst, bld, c),
widths[c] ? offset(src, bld, c)
: fs_reg(brw_imm_ud(pad[c])));
return dst;
}
}
}
namespace brw {
namespace image_access {
/**
* Load a vector from a surface of the given format and dimensionality
* at the given coordinates. \p surf_dims and \p arr_dims give the
* number of non-array and array coordinates of the image respectively.
*/
fs_reg
emit_image_load(const fs_builder &bld,
const fs_reg &image, const fs_reg &addr,
unsigned surf_dims, unsigned arr_dims,
unsigned gl_format)
{
using namespace image_format_info;
using namespace image_format_conversion;
using namespace image_validity;
using namespace image_coordinates;
using namespace surface_access;
const gen_device_info *devinfo = bld.shader->devinfo;
const isl_format format = isl_format_for_gl_format(gl_format);
const isl_format lower_format =
isl_lower_storage_image_format(devinfo, format);
fs_reg tmp;
/* Transform the image coordinates into actual surface coordinates. */
const fs_reg saddr =
emit_image_coordinates(bld, addr, surf_dims, arr_dims, format);
const unsigned dims =
num_image_coordinates(bld, surf_dims, arr_dims, format);
if (isl_has_matching_typed_storage_image_format(devinfo, format)) {
/* Hopefully we get here most of the time... */
tmp = emit_typed_read(bld, image, saddr, dims,
isl_format_get_num_channels(lower_format));
} else {
/* Untyped surface reads return 32 bits of the surface per
* component, without any sort of unpacking or type conversion,
*/
const unsigned size = isl_format_get_layout(format)->bpb / 32;
/* they don't properly handle out of bounds access, so we have to
* check manually if the coordinates are valid and predicate the
* surface read on the result,
*/
const brw_predicate pred =
emit_untyped_image_check(bld, image,
emit_bounds_check(bld, image,
saddr, dims));
/* and they don't know about surface coordinates, we need to
* convert them to a raw memory offset.
*/
const fs_reg laddr = emit_address_calculation(bld, image, saddr, dims);
tmp = emit_untyped_read(bld, image, laddr, 1, size, pred);
/* An out of bounds surface access should give zero as result. */
for (unsigned c = 0; c < size; ++c)
set_predicate(pred, bld.SEL(offset(tmp, bld, c),
offset(tmp, bld, c), brw_imm_d(0)));
}
/* Set the register type to D instead of UD if the data type is
* represented as a signed integer in memory so that sign extension
* is handled correctly by unpack.
*/
if (needs_sign_extension(format))
tmp = retype(tmp, BRW_REGISTER_TYPE_D);
if (!has_supported_bit_layout(devinfo, format)) {
/* Unpack individual vector components from the bitfield if the
* hardware is unable to do it for us.
*/
if (has_split_bit_layout(devinfo, format))
tmp = emit_pack(bld, tmp, get_bit_shifts(lower_format),
get_bit_widths(lower_format));
else
tmp = emit_unpack(bld, tmp, get_bit_shifts(format),
get_bit_widths(format));
} else if ((needs_sign_extension(format) &&
!is_conversion_trivial(devinfo, format)) ||
has_undefined_high_bits(devinfo, format)) {
/* Perform a trivial unpack even though the bit layout matches in
* order to get the most significant bits of each component
* initialized properly.
*/
tmp = emit_unpack(bld, tmp, color_u(0, 32, 64, 96),
get_bit_widths(format));
}
if (!isl_format_has_int_channel(format)) {
if (is_conversion_trivial(devinfo, format)) {
/* Just need to cast the vector to the target type. */
tmp = retype(tmp, BRW_REGISTER_TYPE_F);
} else {
/* Do the right sort of type conversion to float. */
if (isl_format_has_float_channel(format))
tmp = emit_convert_from_float(
bld, tmp, get_bit_widths(format));
else
tmp = emit_convert_from_scaled(
bld, tmp, get_bit_widths(format),
isl_format_has_snorm_channel(format));
}
}
/* Initialize missing components of the result. */
return emit_pad(bld, tmp, get_bit_widths(format));
}
/**
* Store a vector in a surface of the given format and dimensionality at
* the given coordinates. \p surf_dims and \p arr_dims give the number
* of non-array and array coordinates of the image respectively.
*/
void
emit_image_store(const fs_builder &bld, const fs_reg &image,
const fs_reg &addr, const fs_reg &src,
unsigned surf_dims, unsigned arr_dims,
unsigned gl_format)
{
using namespace image_format_info;
using namespace image_format_conversion;
using namespace image_validity;
using namespace image_coordinates;
using namespace surface_access;
const isl_format format = isl_format_for_gl_format(gl_format);
const gen_device_info *devinfo = bld.shader->devinfo;
/* Transform the image coordinates into actual surface coordinates. */
const fs_reg saddr =
emit_image_coordinates(bld, addr, surf_dims, arr_dims, format);
const unsigned dims =
num_image_coordinates(bld, surf_dims, arr_dims, format);
if (gl_format == GL_NONE) {
/* We don't know what the format is, but that's fine because it
* implies write-only access, and typed surface writes are always
* able to take care of type conversion and packing for us.
*/
emit_typed_write(bld, image, saddr, src, dims, 4);
} else {
const isl_format lower_format =
isl_lower_storage_image_format(devinfo, format);
fs_reg tmp = src;
if (!is_conversion_trivial(devinfo, format)) {
/* Do the right sort of type conversion. */
if (isl_format_has_float_channel(format))
tmp = emit_convert_to_float(bld, tmp, get_bit_widths(format));
else if (isl_format_has_int_channel(format))
tmp = emit_convert_to_integer(bld, tmp, get_bit_widths(format),
isl_format_has_sint_channel(format));
else
tmp = emit_convert_to_scaled(bld, tmp, get_bit_widths(format),
isl_format_has_snorm_channel(format));
}
/* We're down to bit manipulation at this point. */
tmp = retype(tmp, BRW_REGISTER_TYPE_UD);
if (!has_supported_bit_layout(devinfo, format)) {
/* Pack the vector components into a bitfield if the hardware
* is unable to do it for us.
*/
if (has_split_bit_layout(devinfo, format))
tmp = emit_unpack(bld, tmp, get_bit_shifts(lower_format),
get_bit_widths(lower_format));
else
tmp = emit_pack(bld, tmp, get_bit_shifts(format),
get_bit_widths(format));
}
if (isl_has_matching_typed_storage_image_format(devinfo, format)) {
/* Hopefully we get here most of the time... */
emit_typed_write(bld, image, saddr, tmp, dims,
isl_format_get_num_channels(lower_format));
} else {
/* Untyped surface writes store 32 bits of the surface per
* component, without any sort of packing or type conversion,
*/
const unsigned size = isl_format_get_layout(format)->bpb / 32;
/* they don't properly handle out of bounds access, so we have
* to check manually if the coordinates are valid and predicate
* the surface write on the result,
*/
const brw_predicate pred =
emit_untyped_image_check(bld, image,
emit_bounds_check(bld, image,
saddr, dims));
/* and, phew, they don't know about surface coordinates, we
* need to convert them to a raw memory offset.
*/
const fs_reg laddr = emit_address_calculation(
bld, image, saddr, dims);
emit_untyped_write(bld, image, laddr, tmp, 1, size, pred);
}
}
}
/**
* Perform an atomic read-modify-write operation in a surface of the
* given dimensionality at the given coordinates. \p surf_dims and \p
* arr_dims give the number of non-array and array coordinates of the
* image respectively. Main building block of the imageAtomic GLSL
* built-ins.
*/
fs_reg
emit_image_atomic(const fs_builder &bld,
const fs_reg &image, const fs_reg &addr,
const fs_reg &src0, const fs_reg &src1,
unsigned surf_dims, unsigned arr_dims,
unsigned rsize, unsigned op)
{
using namespace image_validity;
using namespace image_coordinates;
using namespace surface_access;
/* Avoid performing an atomic operation on an unbound surface. */
const brw_predicate pred = emit_typed_atomic_check(bld, image);
/* Transform the image coordinates into actual surface coordinates. */
const fs_reg saddr =
emit_image_coordinates(bld, addr, surf_dims, arr_dims,
ISL_FORMAT_R32_UINT);
const unsigned dims =
num_image_coordinates(bld, surf_dims, arr_dims,
ISL_FORMAT_R32_UINT);
/* Thankfully we can do without untyped atomics here. */
const fs_reg tmp = emit_typed_atomic(bld, image, saddr, src0, src1,
dims, rsize, op, pred);
/* An unbound surface access should give zero as result. */
if (rsize && pred)
set_predicate(pred, bld.SEL(tmp, tmp, brw_imm_d(0)));
return retype(tmp, src0.type);
}
}
}