/*
 * hextile.c
 *
 * Routines to implement Hextile Encoding
 */

/*
 *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
 *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.
 *  All Rights Reserved.
 *
 *  This is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This software is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this software; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
 *  USA.
 */

/* the hextile encoder is this file works just like the original  I've just
 * tidied the code a bit and renamed a lot of things to integrate it with
 * the vnc server
 */

#include <stdio.h>

#include "vncserv.h"
#include "proto.h"
#include "vncbuffer.h"
#include "pixtrans.h"
#include "profiling.h"
#include "hextile.h"


static void send_hextile32(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp);
static void test_colours32(const CARD32 *data, int size, int *mono, int *solid, CARD32 *bg, CARD32 *fg);
static int subrect_encode32(int w, int h, int mono, hextile_temp_buffer *temp);

static void send_hextile16(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp);
static void test_colours16(const CARD16 *data, int size, int *mono, int *solid, CARD16 *bg, CARD16 *fg);
static int subrect_encode16(int w, int h, int mono, hextile_temp_buffer *temp);

static void send_hextile8(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp);
static void test_colours8(const CARD8 *data, int size, int *mono, int *solid, CARD8 *bg, CARD8 *fg);
static int subrect_encode8(int w, int h, int mono, hextile_temp_buffer *temp);

#define HEXTILE_RAW                     (1 << 0)
#define HEXTILE_BACKGROUNDSPECIFIED     (1 << 1)
#define HEXTILE_FOREGROUNDSPECIFIED     (1 << 2)
#define HEXTILE_ANYSUBRECTS             (1 << 3)
#define HEXTILE_SUBRECTSCOLOURED        (1 << 4)


/*
 *
 */


int hextile_num_rects(vncserv *serv, const area *rect)
{
  return 1;
}

bool hextile_send(vncserv *serv, send_rect_state *state, hextile_temp_buffer *temp)
{
  int space = vncserv_get_tx_space(serv);
  if (space < sizeof(temp->output) + sizeof(rect_header))
  {
    return false;
  }

  /* Write rect header if this is the first call */
  if (state->x == -1)
  {
    rect_header rect;
    rect.x = Swap16IfLE(state->rect.x);
    rect.y = Swap16IfLE(state->rect.y);
    rect.w = Swap16IfLE(state->rect.w);
    rect.h = Swap16IfLE(state->rect.h);
    rect.enc = Swap32IfLE(HEXTILE_ENCODING);
    vncserv_write(serv, &rect, sizeof(rect));
    space -= sizeof(rect);

    state->x = state->rect.x;
    state->y = state->rect.y;

    temp->validfg = temp->validbg = false;
  }

  PROFILE_FUNC_BEGIN

  int max_tiles = 16;

  vncbuffer_reset();

  /* Split into tiles */
  int tile_h = (state->rect.y + state->rect.h) - state->y;
  if (tile_h > 16)
  {
    tile_h = 16;
  }
  while ((space >= sizeof(temp->output)) && max_tiles--)
  {
    int tile_w = (state->rect.x + state->rect.w) - state->x;
    if (tile_w > 16)
    {
      tile_w = 16;
    }

    PROFILE_FUNC_METRIC(tile_w*tile_h)

    switch (serv->clientformat.bits_per_pixel) {
    case 8:
      send_hextile8(serv, state->x, state->y, tile_w, tile_h, temp);
      break;
    case 16:
      send_hextile16(serv, state->x, state->y, tile_w, tile_h, temp);
      break;
    case 32:
      send_hextile32(serv, state->x, state->y, tile_w, tile_h, temp);
      break;
    }

    state->x += tile_w;
    if (state->x == state->rect.x + state->rect.w)
    {
      state->y += tile_h;
      if (state->y == state->rect.y + state->rect.h)
      {
        break;
      }
      state->x = state->rect.x;
      tile_h = (state->rect.y + state->rect.h) - state->y;
      if (tile_h > 16)
      {
        tile_h = 16;
      }
    }

    space = vncserv_get_tx_space(serv) - vncbuffer_used();
  }

  vncbuffer_flush(serv);

  PROFILE_FUNC_END

  return true;
}


/*
 * rfbSendHextiles 32 bpp
 */

static void send_hextile32(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp) {
  CARD32 newbg, newfg;
  int mono, solid;

  CARD32 *clientblock = temp->_32.clientblock;

  pixtrans_do_it(serv, x, y, w, h, clientblock);

  // reset buffer at the start of each 16x16 block, write at the end
  temp->output[0] = 0;
  temp->len = 1;

  test_colours32(clientblock, w*h, &mono, &solid, &newbg, &newfg);

  if (!temp->validbg || (newbg != temp->_32.bg)) {
    temp->validbg = true;
    temp->_32.bg = newbg;
    temp->output[0] |= HEXTILE_BACKGROUNDSPECIFIED;
    temp->output[temp->len++] = ((char*)&(temp->_32.bg))[0];
    temp->output[temp->len++] = ((char*)&(temp->_32.bg))[1];
    temp->output[temp->len++] = ((char*)&(temp->_32.bg))[2];
    temp->output[temp->len++] = ((char*)&(temp->_32.bg))[3];
  }

  if (solid) {
    vncbuffer_write(temp->output, temp->len, serv);
    return;
  }

  temp->output[0] |= HEXTILE_ANYSUBRECTS;

  if (mono) {
    if (!temp->validfg || (newfg != temp->_32.fg)) {
      temp->validfg = true;
      temp->_32.fg = newfg;
      temp->output[0] |= HEXTILE_FOREGROUNDSPECIFIED;
      temp->output[temp->len++] = ((char*)&(temp->_32.fg))[0];
      temp->output[temp->len++] = ((char*)&(temp->_32.fg))[1];
      temp->output[temp->len++] = ((char*)&(temp->_32.fg))[2];
      temp->output[temp->len++] = ((char*)&(temp->_32.fg))[3];
    }
  } else {
    temp->validfg = false;
    temp->output[0] |= HEXTILE_SUBRECTSCOLOURED;
  }

  if (!subrect_encode32(w, h, mono, temp)) {
    // encoding was too large, use raw
    temp->validbg = temp->validfg = false;
    temp->output[0] = HEXTILE_RAW;
    pixtrans_do_it(serv, x, y, w, h, temp->output+1);
    temp->len = sizeof(CARD32)*w*h + 1;
  }

  vncbuffer_write(temp->output, temp->len, serv);
}


static int subrect_encode32(int w, int h, int mono, hextile_temp_buffer *temp) {
  CARD32 *data = temp->_32.clientblock;
  CARD32 bg = temp->_32.bg;
  CARD32 cl2;
  int x, y;
  int i, j;
  int hx = 0, hy, vx = 0, vy;
  int hyflag;
  CARD32 *seg;
  CARD32 *line;
  int hw, hh, vw, vh;
  int thex, they, thew, theh;
  int numsubs = 0;
  int numsubrectslen;

  numsubrectslen = temp->len;
  temp->len++;

  for (y = 0; y < h; y++) {
    line = data + y*w;
    for (x = 0; x < w; x++) {
      if (line[x] != bg) {
        cl2 = line[x];
        hy = y-1;
        hyflag = 1;
        for (j = y; j < h; j++) {
          seg = data+(j*w);
          if (seg[x] != cl2)   break;
          i = x;
          while ((seg[i] == cl2) && (i < w)) i += 1;
          i -= 1;
          if (j == y) vx = hx = i;
          if (i < vx) vx = i;
          if ((hyflag > 0) && (i >= hx))
            hy += 1;
          else
            hyflag = 0;
        }
        vy = j-1;

        /* We now have two possible subrects: (x,y,hx,hy) and
         * (x,y,vx,vy).  We'll choose the bigger of the two.
         */
        hw = hx-x+1;
        hh = hy-y+1;
        vw = vx-x+1;
        vh = vy-y+1;

        thex = x;
        they = y;

        if ((hw*hh) > (vw*vh)) {
          thew = hw;
          theh = hh;
        } else {
          thew = vw;
          theh = vh;
        }

        if (temp->len - numsubrectslen + 2*sizeof(CARD32) > sizeof(CARD32)*w*h)
        {
          return 0;
        }

        numsubs += 1;

        if (!mono) {
          temp->output[temp->len++] = ((char*)&(cl2))[0];
          temp->output[temp->len++] = ((char*)&(cl2))[1];
          temp->output[temp->len++] = ((char*)&(cl2))[2];
          temp->output[temp->len++] = ((char*)&(cl2))[3];
        }

        temp->output[temp->len++] = (thex<<4)     | they;
        temp->output[temp->len++] = ((thew-1)<<4) | (theh-1);

        /*
         * Now mark the subrect as done.
         */
        for (j = they; j < (they+theh); j++)
          for (i = thex; i < (thex+thew); i++)
            data[j*w+i] = bg;
      }
    }
  }
  temp->output[numsubrectslen] = numsubs;

  return 1;
}


/*
 * test_colours() tests if there are one (solid), two (mono) or more
 * colours in a tile and gets a reasonable guess at the best background
 * pixel, and the foreground pixel for mono.
 */

static void test_colours32(const CARD32 *data, int size, int *mono, int *solid, CARD32 *bg, CARD32 *fg) {
  CARD32 colour1 = 0, colour2 = 0;
  int n1 = 0, n2 = 0;

  *mono = 1;
  *solid = 1;

  for (; size > 0; size--, data++) {

    if (n1 == 0)
      colour1 = *data;

    if (*data == colour1) {
      n1++;
      continue;
    }

    if (n2 == 0) {
      *solid = 0;
      colour2 = *data;
    }

    if (*data == colour2) {
      n2++;
      continue;
    }

    *mono = 0;
    break;
  }

  if (n1 > n2) {
    *bg = colour1;
    *fg = colour2;
  } else {
    *bg = colour2;
    *fg = colour1;
  }
}



/*
 * rfbSendHextiles 16 bpp
 */

static void send_hextile16(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp) {
  CARD16 newbg, newfg;
  int mono, solid;

  CARD16 *clientblock = temp->_16.clientblock;

  pixtrans_do_it(serv, x, y, w, h, clientblock);

  // reset buffer at the start of each 16x16 block, write at the end
  temp->output[0] = 0;
  temp->len = 1;

  test_colours16(clientblock, w*h, &mono, &solid, &newbg, &newfg);

  if (!temp->validbg || (newbg != temp->_16.bg)) {
    temp->validbg = true;
    temp->_16.bg = newbg;
    temp->output[0] |= HEXTILE_BACKGROUNDSPECIFIED;
    temp->output[temp->len++] = ((char*)&(temp->_16.bg))[0];
    temp->output[temp->len++] = ((char*)&(temp->_16.bg))[1];
  }

  if (solid) {
    vncbuffer_write(temp->output, temp->len, serv);
    return;
  }

  temp->output[0] |= HEXTILE_ANYSUBRECTS;

  if (mono) {
    if (!temp->validfg || (newfg != temp->_16.fg)) {
      temp->validfg = true;
      temp->_16.fg = newfg;
      temp->output[0] |= HEXTILE_FOREGROUNDSPECIFIED;
      temp->output[temp->len++] = ((char*)&(temp->_16.fg))[0];
      temp->output[temp->len++] = ((char*)&(temp->_16.fg))[1];
    }
  } else {
    temp->validfg = false;
    temp->output[0] |= HEXTILE_SUBRECTSCOLOURED;
  }

  if (!subrect_encode16(w, h, mono, temp)) {
    // encoding was too large, use raw
    temp->validbg = temp->validfg = false;
    temp->output[0] = HEXTILE_RAW;
    pixtrans_do_it(serv, x, y, w, h, temp->output+1);
    temp->len = sizeof(CARD16)*w*h + 1;
  }

  vncbuffer_write(temp->output, temp->len, serv);
}


static int subrect_encode16(int w, int h, int mono, hextile_temp_buffer *temp) {
  CARD16 *data = temp->_16.clientblock;
  CARD16 bg = temp->_16.bg;
  CARD16 cl2;
  int x, y;
  int i, j;
  int hx = 0, hy, vx = 0, vy;
  int hyflag;
  CARD16 *seg;
  CARD16 *line;
  int hw, hh, vw, vh;
  int thex, they, thew, theh;
  int numsubs = 0;
  int numsubrectslen;

  numsubrectslen = temp->len;
  temp->len++;

  for (y = 0; y < h; y++) {
    line = data + y*w;
    for (x = 0; x < w; x++) {
      if (line[x] != bg) {
        cl2 = line[x];
        hy = y-1;
        hyflag = 1;
        for (j = y; j < h; j++) {
          seg = data+(j*w);
          if (seg[x] != cl2)   break;
          i = x;
          while ((seg[i] == cl2) && (i < w)) i += 1;
          i -= 1;
          if (j == y) vx = hx = i;
          if (i < vx) vx = i;
          if ((hyflag > 0) && (i >= hx))
            hy += 1;
          else
            hyflag = 0;
        }
        vy = j-1;

        /* We now have two possible subrects: (x,y,hx,hy) and
         * (x,y,vx,vy).  We'll choose the bigger of the two.
         */
        hw = hx-x+1;
        hh = hy-y+1;
        vw = vx-x+1;
        vh = vy-y+1;

        thex = x;
        they = y;

        if ((hw*hh) > (vw*vh)) {
          thew = hw;
          theh = hh;
        } else {
          thew = vw;
          theh = vh;
        }

        if (temp->len - numsubrectslen + 2*sizeof(CARD16) > sizeof(CARD16)*w*h)
        {
          return 0;
        }

        numsubs += 1;

        if (!mono) {
          temp->output[temp->len++] = ((char*)&(cl2))[0];
          temp->output[temp->len++] = ((char*)&(cl2))[1];
        }

        temp->output[temp->len++] = (thex<<4)     | they;
        temp->output[temp->len++] = ((thew-1)<<4) | (theh-1);

        /*
         * Now mark the subrect as done.
         */
        for (j = they; j < (they+theh); j++)
          for (i = thex; i < (thex+thew); i++)
            data[j*w+i] = bg;
      }
    }
  }
  temp->output[numsubrectslen] = numsubs;

  return 1;
}


/*
 * test_colours() tests if there are one (solid), two (mono) or more
 * colours in a tile and gets a reasonable guess at the best background
 * pixel, and the foreground pixel for mono.
 */

static void test_colours16(const CARD16 *data, int size, int *mono, int *solid, CARD16 *bg, CARD16 *fg) {
  CARD16 colour1 = 0, colour2 = 0;
  int n1 = 0, n2 = 0;

  *mono = 1;
  *solid = 1;

  for (; size > 0; size--, data++) {

    if (n1 == 0)
      colour1 = *data;

    if (*data == colour1) {
      n1++;
      continue;
    }

    if (n2 == 0) {
      *solid = 0;
      colour2 = *data;
    }

    if (*data == colour2) {
      n2++;
      continue;
    }

    *mono = 0;
    break;
  }

  if (n1 > n2) {
    *bg = colour1;
    *fg = colour2;
  } else {
    *bg = colour2;
    *fg = colour1;
  }
}



/*
 * rfbSendHextiles 8 bpp
 */

static void send_hextile8(vncserv *serv, int x, int y, int w, int h, hextile_temp_buffer *temp) {
  CARD8 newbg, newfg;
  int mono, solid;

  CARD8 *clientblock = temp->_8.clientblock;

  pixtrans_do_it(serv, x, y, w, h, clientblock);

  // reset buffer at the start of each 16x16 block, write at the end
  temp->output[0] = 0;
  temp->len = 1;

  test_colours8(clientblock, w*h, &mono, &solid, &newbg, &newfg);

  if (!temp->validbg || (newbg != temp->_8.bg)) {
    temp->validbg = true;
    temp->_8.bg = newbg;
    temp->output[0] |= HEXTILE_BACKGROUNDSPECIFIED;
    temp->output[temp->len++] = temp->_8.bg;
  }

  if (solid) {
    vncbuffer_write(temp->output, temp->len, serv);
    return;
  }

  temp->output[0] |= HEXTILE_ANYSUBRECTS;

  if (mono) {
    if (!temp->validfg || (newfg != temp->_8.fg)) {
      temp->validfg = true;
      temp->_8.fg = newfg;
      temp->output[0] |= HEXTILE_FOREGROUNDSPECIFIED;
      temp->output[temp->len++] = temp->_8.fg;
    }
  } else {
    temp->validfg = false;
    temp->output[0] |= HEXTILE_SUBRECTSCOLOURED;
  }

  if (!subrect_encode8(w, h, mono, temp)) {
    // encoding was too large, use raw
    temp->validbg = temp->validfg = false;
    temp->output[0] = HEXTILE_RAW;
    pixtrans_do_it(serv, x, y, w, h, temp->output+1);
    temp->len = sizeof(CARD8)*w*h + 1;
  }

  vncbuffer_write(temp->output, temp->len, serv);
}


static int subrect_encode8(int w, int h, int mono, hextile_temp_buffer *temp) {
  CARD8 *data = temp->_8.clientblock;
  CARD8 bg = temp->_8.bg;
  CARD8 cl2;
  int x, y;
  int i, j;
  int hx = 0, hy, vx = 0, vy;
  int hyflag;
  CARD8 *seg;
  CARD8 *line;
  int hw, hh, vw, vh;
  int thex, they, thew, theh;
  int numsubs = 0;
  int numsubrectslen;

  numsubrectslen = temp->len;
  temp->len++;

  for (y = 0; y < h; y++) {
    line = data + y*w;
    for (x = 0; x < w; x++) {
      if (line[x] != bg) {
        cl2 = line[x];
        hy = y-1;
        hyflag = 1;
        for (j = y; j < h; j++) {
          seg = data+(j*w);
          if (seg[x] != cl2)   break;
          i = x;
          while ((seg[i] == cl2) && (i < w)) i += 1;
          i -= 1;
          if (j == y) vx = hx = i;
          if (i < vx) vx = i;
          if ((hyflag > 0) && (i >= hx))
            hy += 1;
          else
            hyflag = 0;
        }
        vy = j-1;

        /* We now have two possible subrects: (x,y,hx,hy) and
         * (x,y,vx,vy).  We'll choose the bigger of the two.
         */
        hw = hx-x+1;
        hh = hy-y+1;
        vw = vx-x+1;
        vh = vy-y+1;

        thex = x;
        they = y;

        if ((hw*hh) > (vw*vh)) {
          thew = hw;
          theh = hh;
        } else {
          thew = vw;
          theh = vh;
        }

        if (temp->len - numsubrectslen + 2*sizeof(CARD8) > sizeof(CARD8)*w*h)
        {
          return 0;
        }

        numsubs += 1;

        if (!mono) {
          temp->output[temp->len++] = cl2;
        }

        temp->output[temp->len++] = (thex<<4)     | they;
        temp->output[temp->len++] = ((thew-1)<<4) | (theh-1);

        /*
         * Now mark the subrect as done.
         */
        for (j = they; j < (they+theh); j++)
          for (i = thex; i < (thex+thew); i++)
            data[j*w+i] = bg;
      }
    }
  }
  temp->output[numsubrectslen] = numsubs;

  return 1;
}

/*
 * test_colours() tests if there are one (solid), two (mono) or more
 * colours in a tile and gets a reasonable guess at the best background
 * pixel, and the foreground pixel for mono.
 */

static void test_colours8(const CARD8 *data, int size, int *mono, int *solid, CARD8 *bg, CARD8 *fg) {
  CARD8 colour1 = 0, colour2 = 0;
  int n1 = 0, n2 = 0;

  *mono = 1;
  *solid = 1;

  for (; size > 0; size--, data++) {

    if (n1 == 0)
      colour1 = *data;

    if (*data == colour1) {
      n1++;
      continue;
    }

    if (n2 == 0) {
      *solid = 0;
      colour2 = *data;
    }

    if (*data == colour2) {
      n2++;
      continue;
    }

    *mono = 0;
    break;
  }

  if (n1 > n2) {
    *bg = colour1;
    *fg = colour2;
  } else {
    *bg = colour2;
    *fg = colour1;
  }
}
