/*
* Copyright (c) 2014 Scott Mansell
* Copyright © 2014 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "util/u_blitter.h"
#include "util/u_prim.h"
#include "util/u_format.h"
#include "util/u_pack_color.h"
#include "util/u_upload_mgr.h"
#include "indices/u_primconvert.h"
#include "vc4_context.h"
#include "vc4_resource.h"
#define VC4_HW_2116_COUNT 0x1ef0
static void
vc4_get_draw_cl_space(struct vc4_job *job, int vert_count)
{
/* The SW-5891 workaround may cause us to emit multiple shader recs
* and draw packets.
*/
int num_draws = DIV_ROUND_UP(vert_count, 65535 - 2) + 1;
/* Binner gets our packet state -- vc4_emit.c contents,
* and the primitive itself.
*/
cl_ensure_space(&job->bcl,
256 + (VC4_PACKET_GL_ARRAY_PRIMITIVE_SIZE +
VC4_PACKET_GL_SHADER_STATE_SIZE) * num_draws);
/* Nothing for rcl -- that's covered by vc4_context.c */
/* shader_rec gets up to 12 dwords of reloc handles plus a maximally
* sized shader_rec (104 bytes base for 8 vattrs plus 32 bytes of
* vattr stride).
*/
cl_ensure_space(&job->shader_rec,
(12 * sizeof(uint32_t) + 104 + 8 * 32) * num_draws);
/* Uniforms are covered by vc4_write_uniforms(). */
/* There could be up to 16 textures per stage, plus misc other
* pointers.
*/
cl_ensure_space(&job->bo_handles, (2 * 16 + 20) * sizeof(uint32_t));
cl_ensure_space(&job->bo_pointers,
(2 * 16 + 20) * sizeof(struct vc4_bo *));
}
/**
* Does the initial bining command list setup for drawing to a given FBO.
*/
static void
vc4_start_draw(struct vc4_context *vc4)
{
struct vc4_job *job = vc4->job;
if (job->needs_flush)
return;
vc4_get_draw_cl_space(job, 0);
cl_emit(&job->bcl, TILE_BINNING_MODE_CONFIGURATION, bin) {
bin.width_in_tiles = job->draw_tiles_x;
bin.height_in_tiles = job->draw_tiles_y;
bin.multisample_mode_4x = job->msaa;
}
/* START_TILE_BINNING resets the statechange counters in the hardware,
* which are what is used when a primitive is binned to a tile to
* figure out what new state packets need to be written to that tile's
* command list.
*/
cl_emit(&job->bcl, START_TILE_BINNING, start);
/* Reset the current compressed primitives format. This gets modified
* by VC4_PACKET_GL_INDEXED_PRIMITIVE and
* VC4_PACKET_GL_ARRAY_PRIMITIVE, so it needs to be reset at the start
* of every tile.
*/
cl_emit(&job->bcl, PRIMITIVE_LIST_FORMAT, list) {
list.data_type = _16_BIT_INDEX;
list.primitive_type = TRIANGLES_LIST;
}
job->needs_flush = true;
job->draw_width = vc4->framebuffer.width;
job->draw_height = vc4->framebuffer.height;
}
static void
vc4_predraw_check_textures(struct pipe_context *pctx,
struct vc4_texture_stateobj *stage_tex)
{
struct vc4_context *vc4 = vc4_context(pctx);
for (int i = 0; i < stage_tex->num_textures; i++) {
struct vc4_sampler_view *view =
vc4_sampler_view(stage_tex->textures[i]);
if (!view)
continue;
if (view->texture != view->base.texture)
vc4_update_shadow_baselevel_texture(pctx, &view->base);
vc4_flush_jobs_writing_resource(vc4, view->texture);
}
}
static void
vc4_emit_gl_shader_state(struct vc4_context *vc4,
const struct pipe_draw_info *info,
uint32_t extra_index_bias)
{
struct vc4_job *job = vc4->job;
/* VC4_DIRTY_VTXSTATE */
struct vc4_vertex_stateobj *vtx = vc4->vtx;
/* VC4_DIRTY_VTXBUF */
struct vc4_vertexbuf_stateobj *vertexbuf = &vc4->vertexbuf;
/* The simulator throws a fit if VS or CS don't read an attribute, so
* we emit a dummy read.
*/
uint32_t num_elements_emit = MAX2(vtx->num_elements, 1);
/* Emit the shader record. */
cl_start_shader_reloc(&job->shader_rec, 3 + num_elements_emit);
cl_emit(&job->shader_rec, SHADER_RECORD, rec) {
rec.enable_clipping = true;
/* VC4_DIRTY_COMPILED_FS */
rec.fragment_shader_is_single_threaded =
!vc4->prog.fs->fs_threaded;
/* VC4_DIRTY_PRIM_MODE | VC4_DIRTY_RASTERIZER */
rec.point_size_included_in_shaded_vertex_data =
(info->mode == PIPE_PRIM_POINTS &&
vc4->rasterizer->base.point_size_per_vertex);
/* VC4_DIRTY_COMPILED_FS */
rec.fragment_shader_number_of_varyings =
vc4->prog.fs->num_inputs;
rec.fragment_shader_code_address =
cl_address(vc4->prog.fs->bo, 0);
rec.coordinate_shader_attribute_array_select_bits =
vc4->prog.cs->vattrs_live;
rec.coordinate_shader_total_attributes_size =
vc4->prog.cs->vattr_offsets[8];
rec.coordinate_shader_code_address =
cl_address(vc4->prog.cs->bo, 0);
rec.vertex_shader_attribute_array_select_bits =
vc4->prog.vs->vattrs_live;
rec.vertex_shader_total_attributes_size =
vc4->prog.vs->vattr_offsets[8];
rec.vertex_shader_code_address =
cl_address(vc4->prog.vs->bo, 0);
};
uint32_t max_index = 0xffff;
for (int i = 0; i < vtx->num_elements; i++) {
struct pipe_vertex_element *elem = &vtx->pipe[i];
struct pipe_vertex_buffer *vb =
&vertexbuf->vb[elem->vertex_buffer_index];
struct vc4_resource *rsc = vc4_resource(vb->buffer.resource);
/* not vc4->dirty tracked: vc4->last_index_bias */
uint32_t offset = (vb->buffer_offset +
elem->src_offset +
vb->stride * (info->index_bias +
extra_index_bias));
uint32_t vb_size = rsc->bo->size - offset;
uint32_t elem_size =
util_format_get_blocksize(elem->src_format);
cl_emit(&job->shader_rec, ATTRIBUTE_RECORD, attr) {
attr.address = cl_address(rsc->bo, offset);
attr.number_of_bytes_minus_1 = elem_size - 1;
attr.stride = vb->stride;
attr.coordinate_shader_vpm_offset =
vc4->prog.cs->vattr_offsets[i];
attr.vertex_shader_vpm_offset =
vc4->prog.vs->vattr_offsets[i];
}
if (vb->stride > 0) {
max_index = MIN2(max_index,
(vb_size - elem_size) / vb->stride);
}
}
if (vtx->num_elements == 0) {
assert(num_elements_emit == 1);
struct vc4_bo *bo = vc4_bo_alloc(vc4->screen, 4096, "scratch VBO");
cl_emit(&job->shader_rec, ATTRIBUTE_RECORD, attr) {
attr.address = cl_address(bo, 0);
attr.number_of_bytes_minus_1 = 16 - 1;
attr.stride = 0;
attr.coordinate_shader_vpm_offset = 0;
attr.vertex_shader_vpm_offset = 0;
}
}
cl_emit(&job->bcl, GL_SHADER_STATE, shader_state) {
/* Note that number of attributes == 0 in the packet means 8
* attributes. This field also contains the offset into
* shader_rec.
*/
assert(vtx->num_elements <= 8);
shader_state.number_of_attribute_arrays =
num_elements_emit & 0x7;
}
vc4_write_uniforms(vc4, vc4->prog.fs,
&vc4->constbuf[PIPE_SHADER_FRAGMENT],
&vc4->fragtex);
vc4_write_uniforms(vc4, vc4->prog.vs,
&vc4->constbuf[PIPE_SHADER_VERTEX],
&vc4->verttex);
vc4_write_uniforms(vc4, vc4->prog.cs,
&vc4->constbuf[PIPE_SHADER_VERTEX],
&vc4->verttex);
vc4->last_index_bias = info->index_bias + extra_index_bias;
vc4->max_index = max_index;
job->shader_rec_count++;
}
/**
* HW-2116 workaround: Flush the batch before triggering the hardware state
* counter wraparound behavior.
*
* State updates are tracked by a global counter which increments at the first
* state update after a draw or a START_BINNING. Tiles can then have their
* state updated at draw time with a set of cheap checks for whether the
* state's copy of the global counter matches the global counter the last time
* that state was written to the tile.
*
* The state counters are relatively small and wrap around quickly, so you
* could get false negatives for needing to update a particular state in the
* tile. To avoid this, the hardware attempts to write all of the state in
* the tile at wraparound time. This apparently is broken, so we just flush
* everything before that behavior is triggered. A batch flush is sufficient
* to get our current contents drawn and reset the counters to 0.
*
* Note that we can't just use VC4_PACKET_FLUSH_ALL, because that caps the
* tiles with VC4_PACKET_RETURN_FROM_LIST.
*/
static void
vc4_hw_2116_workaround(struct pipe_context *pctx, int vert_count)
{
struct vc4_context *vc4 = vc4_context(pctx);
struct vc4_job *job = vc4_get_job_for_fbo(vc4);
if (job->draw_calls_queued + vert_count / 65535 >= VC4_HW_2116_COUNT) {
perf_debug("Flushing batch due to HW-2116 workaround "
"(too many draw calls per scene\n");
vc4_job_submit(vc4, job);
}
}
static void
vc4_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info)
{
struct vc4_context *vc4 = vc4_context(pctx);
struct pipe_draw_info local_info;
if (!info->count_from_stream_output && !info->indirect &&
!info->primitive_restart &&
!u_trim_pipe_prim(info->mode, (unsigned*)&info->count))
return;
if (info->mode >= PIPE_PRIM_QUADS) {
if (info->mode == PIPE_PRIM_QUADS &&
info->count == 4 &&
!vc4->rasterizer->base.flatshade) {
local_info = *info;
local_info.mode = PIPE_PRIM_TRIANGLE_FAN;
info = &local_info;
} else {
util_primconvert_save_rasterizer_state(vc4->primconvert, &vc4->rasterizer->base);
util_primconvert_draw_vbo(vc4->primconvert, info);
perf_debug("Fallback conversion for %d %s vertices\n",
info->count, u_prim_name(info->mode));
return;
}
}
/* Before setting up the draw, do any fixup blits necessary. */
vc4_predraw_check_textures(pctx, &vc4->verttex);
vc4_predraw_check_textures(pctx, &vc4->fragtex);
vc4_hw_2116_workaround(pctx, info->count);
struct vc4_job *job = vc4_get_job_for_fbo(vc4);
/* Make sure that the raster order flags haven't changed, which can
* only be set at job granularity.
*/
if (job->flags != vc4->rasterizer->tile_raster_order_flags) {
vc4_job_submit(vc4, job);
job = vc4_get_job_for_fbo(vc4);
}
vc4_get_draw_cl_space(job, info->count);
if (vc4->prim_mode != info->mode) {
vc4->prim_mode = info->mode;
vc4->dirty |= VC4_DIRTY_PRIM_MODE;
}
vc4_start_draw(vc4);
if (!vc4_update_compiled_shaders(vc4, info->mode)) {
debug_warn_once("shader compile failed, skipping draw call.\n");
return;
}
vc4_emit_state(pctx);
bool needs_drawarrays_shader_state = false;
if ((vc4->dirty & (VC4_DIRTY_VTXBUF |
VC4_DIRTY_VTXSTATE |
VC4_DIRTY_PRIM_MODE |
VC4_DIRTY_RASTERIZER |
VC4_DIRTY_COMPILED_CS |
VC4_DIRTY_COMPILED_VS |
VC4_DIRTY_COMPILED_FS |
vc4->prog.cs->uniform_dirty_bits |
vc4->prog.vs->uniform_dirty_bits |
vc4->prog.fs->uniform_dirty_bits)) ||
vc4->last_index_bias != info->index_bias) {
if (info->index_size)
vc4_emit_gl_shader_state(vc4, info, 0);
else
needs_drawarrays_shader_state = true;
}
vc4->dirty = 0;
/* Note that the primitive type fields match with OpenGL/gallium
* definitions, up to but not including QUADS.
*/
if (info->index_size) {
uint32_t index_size = info->index_size;
uint32_t offset = info->start * index_size;
struct pipe_resource *prsc;
if (info->index_size == 4) {
prsc = vc4_get_shadow_index_buffer(pctx, info,
offset,
info->count, &offset);
index_size = 2;
} else {
if (info->has_user_indices) {
prsc = NULL;
u_upload_data(vc4->uploader, 0,
info->count * index_size, 4,
info->index.user,
&offset, &prsc);
} else {
prsc = info->index.resource;
}
}
struct vc4_resource *rsc = vc4_resource(prsc);
struct vc4_cl_out *bcl = cl_start(&job->bcl);
/* The original design for the VC4 kernel UABI had multiple
* packets that used relocations in the BCL (some of which
* needed two BOs), but later modifications eliminated all but
* this one usage. We have an arbitrary 32-bit offset value,
* and need to also supply an arbitrary 32-bit index buffer
* GEM handle, so we have this fake packet we emit in our BCL
* to be validated, which the kernel uses at validation time
* to perform the relocation in the IB packet (without
* emitting to the actual HW).
*/
uint32_t hindex = vc4_gem_hindex(job, rsc->bo);
if (job->last_gem_handle_hindex != hindex) {
cl_u8(&bcl, VC4_PACKET_GEM_HANDLES);
cl_u32(&bcl, hindex);
cl_u32(&bcl, 0);
job->last_gem_handle_hindex = hindex;
}
cl_u8(&bcl, VC4_PACKET_GL_INDEXED_PRIMITIVE);
cl_u8(&bcl,
info->mode |
(index_size == 2 ?
VC4_INDEX_BUFFER_U16:
VC4_INDEX_BUFFER_U8));
cl_u32(&bcl, info->count);
cl_u32(&bcl, offset);
cl_u32(&bcl, vc4->max_index);
cl_end(&job->bcl, bcl);
job->draw_calls_queued++;
if (info->index_size == 4 || info->has_user_indices)
pipe_resource_reference(&prsc, NULL);
} else {
uint32_t count = info->count;
uint32_t start = info->start;
uint32_t extra_index_bias = 0;
static const uint32_t max_verts = 65535;
/* GFXH-515 / SW-5891: The binner emits 16 bit indices for
* drawarrays, which means that if start + count > 64k it
* would truncate the top bits. Work around this by emitting
* a limited number of primitives at a time and reemitting the
* shader state pointing farther down the vertex attribute
* arrays.
*
* To do this properly for line loops or trifans, we'd need to
* make a new VB containing the first vertex plus whatever
* remainder.
*/
if (start + count > max_verts) {
extra_index_bias = start;
start = 0;
needs_drawarrays_shader_state = true;
}
while (count) {
uint32_t this_count = count;
uint32_t step = count;
if (needs_drawarrays_shader_state) {
vc4_emit_gl_shader_state(vc4, info,
extra_index_bias);
}
if (count > max_verts) {
switch (info->mode) {
case PIPE_PRIM_POINTS:
this_count = step = max_verts;
break;
case PIPE_PRIM_LINES:
this_count = step = max_verts - (max_verts % 2);
break;
case PIPE_PRIM_LINE_STRIP:
this_count = max_verts;
step = max_verts - 1;
break;
case PIPE_PRIM_LINE_LOOP:
this_count = max_verts;
step = max_verts - 1;
debug_warn_once("unhandled line loop "
"looping behavior with "
">65535 verts\n");
break;
case PIPE_PRIM_TRIANGLES:
this_count = step = max_verts - (max_verts % 3);
break;
case PIPE_PRIM_TRIANGLE_STRIP:
this_count = max_verts;
step = max_verts - 2;
break;
default:
debug_warn_once("unhandled primitive "
"max vert count, truncating\n");
this_count = step = max_verts;
}
}
cl_emit(&job->bcl, VERTEX_ARRAY_PRIMITIVES, array) {
array.primitive_mode = info->mode;
array.length = this_count;
array.index_of_first_vertex = start;
}
job->draw_calls_queued++;
count -= step;
extra_index_bias += start + step;
start = 0;
needs_drawarrays_shader_state = true;
}
}
/* We shouldn't have tripped the HW_2116 bug with the GFXH-515
* workaround.
*/
assert(job->draw_calls_queued <= VC4_HW_2116_COUNT);
if (vc4->zsa && vc4->framebuffer.zsbuf) {
struct vc4_resource *rsc =
vc4_resource(vc4->framebuffer.zsbuf->texture);
if (vc4->zsa->base.depth.enabled) {
job->resolve |= PIPE_CLEAR_DEPTH;
rsc->initialized_buffers = PIPE_CLEAR_DEPTH;
}
if (vc4->zsa->base.stencil[0].enabled) {
job->resolve |= PIPE_CLEAR_STENCIL;
rsc->initialized_buffers |= PIPE_CLEAR_STENCIL;
}
}
job->resolve |= PIPE_CLEAR_COLOR0;
/* If we've used half of the presumably 256MB CMA area, flush the job
* so that we don't accumulate a job that will end up not being
* executable.
*/
if (job->bo_space > 128 * 1024 * 1024)
vc4_flush(pctx);
if (vc4_debug & VC4_DEBUG_ALWAYS_FLUSH)
vc4_flush(pctx);
}
static uint32_t
pack_rgba(enum pipe_format format, const float *rgba)
{
union util_color uc;
util_pack_color(rgba, format, &uc);
if (util_format_get_blocksize(format) == 2)
return uc.us;
else
return uc.ui[0];
}
static void
vc4_clear(struct pipe_context *pctx, unsigned buffers,
const union pipe_color_union *color, double depth, unsigned stencil)
{
struct vc4_context *vc4 = vc4_context(pctx);
struct vc4_job *job = vc4_get_job_for_fbo(vc4);
if (buffers & PIPE_CLEAR_DEPTHSTENCIL) {
struct vc4_resource *rsc =
vc4_resource(vc4->framebuffer.zsbuf->texture);
unsigned zsclear = buffers & PIPE_CLEAR_DEPTHSTENCIL;
/* Clearing ZS will clear both Z and stencil, so if we're
* trying to clear just one then we need to draw a quad to do
* it instead. We need to do this before setting up
* tile-based clears in vc4->job, because the blitter may
* submit the current job.
*/
if ((zsclear == PIPE_CLEAR_DEPTH ||
zsclear == PIPE_CLEAR_STENCIL) &&
(rsc->initialized_buffers & ~(zsclear | job->cleared)) &&
util_format_is_depth_and_stencil(vc4->framebuffer.zsbuf->format)) {
static const union pipe_color_union dummy_color = {};
perf_debug("Partial clear of Z+stencil buffer, "
"drawing a quad instead of fast clearing\n");
vc4_blitter_save(vc4);
util_blitter_clear(vc4->blitter,
vc4->framebuffer.width,
vc4->framebuffer.height,
1,
zsclear,
&dummy_color, depth, stencil);
buffers &= ~zsclear;
if (!buffers)
return;
job = vc4_get_job_for_fbo(vc4);
}
}
/* We can't flag new buffers for clearing once we've queued draws. We
* could avoid this by using the 3d engine to clear.
*/
if (job->draw_calls_queued) {
perf_debug("Flushing rendering to process new clear.\n");
vc4_job_submit(vc4, job);
job = vc4_get_job_for_fbo(vc4);
}
if (buffers & PIPE_CLEAR_COLOR0) {
struct vc4_resource *rsc =
vc4_resource(vc4->framebuffer.cbufs[0]->texture);
uint32_t clear_color;
if (vc4_rt_format_is_565(vc4->framebuffer.cbufs[0]->format)) {
/* In 565 mode, the hardware will be packing our color
* for us.
*/
clear_color = pack_rgba(PIPE_FORMAT_R8G8B8A8_UNORM,
color->f);
} else {
/* Otherwise, we need to do this packing because we
* support multiple swizzlings of RGBA8888.
*/
clear_color =
pack_rgba(vc4->framebuffer.cbufs[0]->format,
color->f);
}
job->clear_color[0] = job->clear_color[1] = clear_color;
rsc->initialized_buffers |= (buffers & PIPE_CLEAR_COLOR0);
}
if (buffers & PIPE_CLEAR_DEPTHSTENCIL) {
struct vc4_resource *rsc =
vc4_resource(vc4->framebuffer.zsbuf->texture);
/* Though the depth buffer is stored with Z in the high 24,
* for this field we just need to store it in the low 24.
*/
if (buffers & PIPE_CLEAR_DEPTH) {
job->clear_depth = util_pack_z(PIPE_FORMAT_Z24X8_UNORM,
depth);
}
if (buffers & PIPE_CLEAR_STENCIL)
job->clear_stencil = stencil;
rsc->initialized_buffers |= (buffers & PIPE_CLEAR_DEPTHSTENCIL);
}
job->draw_min_x = 0;
job->draw_min_y = 0;
job->draw_max_x = vc4->framebuffer.width;
job->draw_max_y = vc4->framebuffer.height;
job->cleared |= buffers;
job->resolve |= buffers;
vc4_start_draw(vc4);
}
static void
vc4_clear_render_target(struct pipe_context *pctx, struct pipe_surface *ps,
const union pipe_color_union *color,
unsigned x, unsigned y, unsigned w, unsigned h,
bool render_condition_enabled)
{
fprintf(stderr, "unimpl: clear RT\n");
}
static void
vc4_clear_depth_stencil(struct pipe_context *pctx, struct pipe_surface *ps,
unsigned buffers, double depth, unsigned stencil,
unsigned x, unsigned y, unsigned w, unsigned h,
bool render_condition_enabled)
{
fprintf(stderr, "unimpl: clear DS\n");
}
void
vc4_draw_init(struct pipe_context *pctx)
{
pctx->draw_vbo = vc4_draw_vbo;
pctx->clear = vc4_clear;
pctx->clear_render_target = vc4_clear_render_target;
pctx->clear_depth_stencil = vc4_clear_depth_stencil;
}