/* Simple BBC News ticker client
 * (c) Darren Salt
 * GPL applies
 * $Id: fetch.c,v 1.16 2005/07/04 00:49:39 ds Exp $
 */

/* System includes */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <errno.h>
#include <swis.h>

#include "sys/types.h"
#include "sys/errno.h"
#include "sys/ioctl.h"
#include "sys/socket.h"
#include "socklib.h"
#include "sys/select.h"
#include "sys/time.h"
#include "unixlib.h"
#include "riscos.h"

#include "netinet/in.h"
#include "arpa/inet.h"

#include "netdb.h"

/* Program includes */

#include "globals.h"

#include "configure.h"
#include "digest.h"
#include "fetch.h"
#include "fetchers.h"
#include "util.h"
#include "wimp.h"
#include "wimpmenu.h"


#define Resolver_GetHost 0x46001


/* Fetched text files */
char *headings_data = 0;
char *stories_data;

/* Proxying? */
static struct
{
  char server[MAX_PROXY_LENGTH];
  ushort port;
  const char *bypass;
  char auth_type;
  char use;
  char *auth;
}
proxy =
{0};

static const char proxy_bypass_msg[] = " (bypassing proxy)";

/* Control */
fetch_status fetch = {0};

static void free_resources_menu (void);


static int
fetch_poll (void)
{
  if (nullpoll ())
    return 1;
  return fetch.aborted;
}


/* Resolve a hostname */

static ulong
resolve (const char *hostname)
{
  struct hostent *hostent = 0;
  struct in_addr inaddr;
  char swiname[256];

  if (inet_aton (hostname, &inaddr) == 1)
    return inaddr.s_addr;

  /* use multitasking resolver if possible */
  if (_swix (OS_SWINumberToString, _INR (0, 2), Resolver_GetHost, swiname,
	     sizeof (swiname) - 1) == NULL
      && !strcmp (swiname, "Resolver_GetHost"))
  {
    do
    {
      int status = 0;
      _kernel_oserror *e = _swix (Resolver_GetHost, _IN (0) | _OUTR (0, 1),
				  hostname, &status, &hostent);
      if (e && e->errnum == 486)
	goto singletask;
      if (e)
      {
	status_oserror ("Resolver error: ", e);
	goto doexit;
      }
      if (status != EINPROGRESS)
      {
	if (hostent)
	  goto finished;
	goto failed;
      }
    }
    while (!fetch_poll ());
    if (fetch.aborted)
      return 0;
    exit (0);
  }

singletask:
  hostent = gethostbyname (hostname);
  if (!hostent)
  {
  failed:
    status_error ("The server or proxy address could not be resolved");
  doexit:
    if (!task_handle)
      exit (2);
    return 0;
  }

finished:
  {
    unsigned int i = hostent->h_length / 4;
    i = ((unsigned int) read_monotonic_time ()) % i;
    return *((int **) hostent->h_addr_list)[i];
  }
}


/* Close a socket. Note that we only expect to have none or one open. */

static int Socket = -1;

static void
socket_close (int socket)
{
  if (socket != -1)
    socketclose (socket);
  if (socket == Socket)
    Socket = -1;
}


/* Shutdown / clear */


static void
fetch_shutdown (void)
{
  socket_close (Socket);
}


/* Open a socket.
 * Try to use the (configured) proxy, but connect directly if that fails.
 */

static int
socket_open (const char server[], ushort port, int i_use_proxy)
{
  int handle, data = 1;
  struct sockaddr_in addr;

  socket_close (Socket);

  Socket = handle = socket (AF_INET, SOCK_STREAM, PF_UNSPEC);
  if (handle == -1)
  {
    status_errno ("Connect failure: ");
    return -1;
  }

  if (ioctl (handle, FIONBIO, &data))
  {
    socket_close (handle);
    status_errno ("Connect failure: ");
    return -1;
  }

  addr.sin_family = AF_INET;

  /* first, try the proxy; fall back on the server itself */
  if (i_use_proxy && proxy.server)
  {
    addr.sin_addr.s_addr = resolve (proxy.server);
    if (!addr.sin_addr.s_addr)
      return -2;
    addr.sin_port = htons (proxy.port);
    connect (handle, (struct sockaddr *) &addr, sizeof (addr));
    if (handle != -1)
      return handle;

    if (proxy.use)
      proxy.bypass = proxy_bypass_msg;
  }

  addr.sin_addr.s_addr = resolve (server);
  if (!addr.sin_addr.s_addr)
    return -2;
  addr.sin_port = htons (port);
  connect (handle, (struct sockaddr *) &addr, sizeof (addr));
  return handle;
}


/* Wait for confirmation or denial of connection through the given socket. */

static int
socket_connected (int handle)
{
  fd_set fds;
  struct timeval timeout;
  int numready;

  timerclear (&timeout);	/* no timeout */
  FD_ZERO (&fds);
  FD_SET (handle, &fds);
  numready = select (handle + 1, NULL, &fds, NULL, &timeout);

  if (numready == -1 || (numready > 0 && FD_ISSET (handle, &fds)))
  {
    int result;
    int size = sizeof (result);

    if (getsockopt (handle, SOL_SOCKET, SO_ERROR,
		    (char *) &result, &size) == -1)
      return errno;

    return result;		/* 0 if connected */
  }

  return -1;			/* not yet connected */
}


/* Write data to the socket. May call the WIMP polling code. */

static int
send_data (int socket, ...)
{
  va_list arg;
  char *text;
  struct iovec *scatter = 0, *sendptr;
  int length = 0, alloc = 0;

  va_start (arg, socket);
  while ((text = va_arg (arg, char *)) != 0)
    if (*text)
    {
      if (length == alloc)
      {
	scatter = realloc (scatter, (alloc += 16) * sizeof (struct iovec));
	if (!scatter)
	{
	  errno = ENOMEM;
	  return -1;
	}
      }
      scatter[length].iov_base = text;
      scatter[length].iov_len = strlen (text);
      length++;
    }

  sendptr = scatter;
  while (length)
  {
    int sent = writev (socket, sendptr, length);
    if (sent < 0)		/* hmm, an error? */
    {
      if (errno == EWOULDBLOCK)
      {
	if (fetch_poll ())	/* yes, but it's harmless; poll, try again */
	{
	  if (fetch.aborted)
	    return -1;
	  socket_close (socket);
	  exit (0);
	}
      }
      status_errno ("Socket write error: ");	/* yes - report and return */
      return -1;
    }
    else
      /* Assumption: sendptr[] content not altered by writev() */
      while (length)
      {
	if (sent < sendptr->iov_len)
	{
	  sendptr->iov_base += sent;
	  sendptr->iov_len -= sent;
	  break;
	}
	sent -= sendptr->iov_len;
	sendptr++;
	length--;
      }
  }

  free (scatter);
  return 0;
}


static char *
base64cat (char *dest, ...)
{
  va_list arg;
  char *ptr = dest + strlen (dest);
  const char *start = ptr;
  const char *text = 0;
  int byte = 0, bit = 8;
  static const char encode[] =
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

  va_start (arg, dest);
  text = va_arg (arg, const char *);
  if (text)
  {
    dest = realloc (dest, strlen (dest) + strlen (text) * 4 / 3 + 5);
    if (!dest)
      return 0;
    for (;;)
    {
      if (bit)
      {
	char c;
	while ((c = *text++) < 32)
	  if ((text = va_arg (arg, const char *)) == 0)
	    goto done;
	byte |= c << bit;
	bit -= 8;
      }
      *ptr++ = encode[(byte >> 10) & 63];
      byte <<= 6;
      bit += 6;
    }
  }
done:
  while (bit < 8)
  {
    *ptr++ = encode[(byte >> 10) & 63];
    byte <<= 6;
    bit += 6;
  }
  while ((ptr - start) & 3)
    *ptr++ = '=';
  *ptr = 0;

  return dest;
}


static void
wipe_auth_password ()
{
  /* Assumption: the window is closed. */
  char *text = get_icon_text (wind.proxy_auth, 6);
  size_t length = get_icon_text_length (wind.proxy_auth, 6);
  memset (text, 0, length);
}


static char *
unctrl (char *p)
{
  char *q = p - 1;
  while (*++q >= ' ')
    ;
  *q = '\0';
  return p;
}


static struct
{
  const char *user, *pass;
}
cache = {0};


static char *
generate_auth (char type, const char *headers, const char *uri, int reset)
{
  char *auth = 0;

  if (cache.user)
  {
    proxy.auth_type = type;
    switch (type)
    {
    case 'b':
      auth = concat (auth, "Proxy-Authorization: Basic ", END);
      if (auth)
	auth = base64cat (auth, cache.user, ":", cache.pass, 0);
      if (auth)
	auth = concat (auth, "\r\n", END);
      return auth;
    case 'd':
      auth = calculate_digest (headers, cache.user, cache.pass, uri, 0, reset);
      return auth;
    }
  }
  return "";
}


static void
get_auth_details (const char *headers, const char *uri)
{
  struct wimp_getwindowstate_block block;
  const char *type = calculate_digest (headers, 0, 0, 0, 1, 0);

  proxy.auth_type = type[0];

  if (type[0] == '?')
  {
    status_error ("Unknown authentication scheme!");
    fetch.aborted = 1;
    return;
  }

  /* Don't require user/pass if we're doing digest and we've been told that
   * the proxy-supplied nonce value (which we used in the generation of our
   * current Proxy-Authorization header) is stale.
   */
  if (type[1] != 's' || proxy.auth_type != type[0])
  {
    const char *p = get_realm ();
    set_icon_text (wind.proxy_auth, 2, p ? p : proxy.server);
    wipe_auth_password ();

    wimp_get_window_state (&block, wind.proxy_auth);
    centre_window (&block);
    block.handle_behind = -1;
    wimp_open_window (&block);
    if (*(get_icon_text (wind.proxy_auth, 4)) >= ' ')
      set_caret_icon (wind.proxy_auth, 6);
    else
      set_caret_icon (wind.proxy_auth, 4);

    do
    {
      if (fetch_poll ())
      {
	if (fetch.aborted)
	{
	  wipe_auth_password ();
	  wimp_close_window ((int *) &block);
	  return;
	}
	exit (0);
      }
      wimp_get_window_state (&block, 0);
    }
    while (block.window_flags & 1 << 16);
  }

  cache.user = unctrl (get_icon_text (wind.proxy_auth, 4));
  cache.pass = unctrl (get_icon_text (wind.proxy_auth, 6));
}


static void
readline (char buffer[], size_t size, int echo)
{
  int l = _swi (OS_ReadLine, _INR (0, 4) | _RETURN (1),
		(ulong) buffer | (echo ? 3UL : 1UL) << 30, size, 32, 255,
		'*');
  buffer[l] = '\0';
}


static void
get_auth_details_singletask (const char *headers, const char *uri)
{
  /* We need to bypass the C output routines here... */
  static char user[32], pass[32];
  const char *p = get_realm ();

  /* NOT IMPLEMENTED: parse the Proxy-Authenticate header. */
  /* NOT IMPLEMENTED: handle digest authentication. */
  proxy.auth_type = 'b';

  _swi (OS_Write0, _IN (1), "Proxy authentication is required for ");
  _swi (OS_Write0, _IN (1), p ? p : proxy.server);
  _swi (OS_NewLine, 0);
  _swi (OS_Write0, _IN (1), "User name: ");
  readline (user, sizeof (user), 1);
  _swi (OS_Write0, _IN (1), "Password: ");
  readline (user, sizeof (user), 0);

  cache.user = user;
  cache.pass = pass;
}


/* Remove all CRs and lose the HTTP headers.
 * Updates *dlen with the new length.
 * Returns the input text pointer.
 */

static char *
process_file (char *data, int length, int *dlen, int lead_lf)
{
  char *p;
  char *content;

  /* eliminate CRs */
  while ((p = memchr (data, '\r', length)) != 0)
    memmove (p, p + 1, --length - (p - data));

  /* find the (end of the) first blank line */
  content = strstr (data, "\n\n");
  content = content ? (content + 2) : (data + length);

  /* extract any charset information */
  strcpy (ticker_data.charset, "iso-8859-1"); /* default */
  p = strstr (data, "\nContent-Type");
  if (p && p < content)
  {
    p = strstr (p, "charset=");
    if (p && (p[-1] == ';' || isspace(p[-1])))
    {
      char *q = strchr (p += 8, '\n');
      if (q > p + 31)
        q = p + 31;
      memcpy (ticker_data.charset, p, q - p);
      ticker_data.charset[q - p] = 0;
    }
  }

  /* eliminate all up to the first blank line,
   * and maybe add three blank lines */
  if (lead_lf)
  {
    length -= content - data - 4;
    memmove (data + 4, content, length);
    data[3] = data[2] = data[1] = data[0] = '\n';
  }
  else
  {
    length -= content - data;
    memmove (data, content, length);
  }
  data = realloc (data, length + 1);

  /* NUL-terminate the text */
  if (data)
    data[length] = 0;

  *dlen = length;
  return data;
}


/* Build an http URL from supplied components. */

static char *
build_url (const char *server, int port, const char *path)
{
  char *url = malloc (strlen (fetcher.server) + strlen (path) + 20);
  if (url)
  {
    if (!port || port == 80)
      sprintf (url, "http://%s/%s", server, path);
    else
      sprintf (url, "http://%s:%i/%s", server, port, path);
  }
  return url;
}


/* Fetch data from the given URL.
 * Note that the server and port must be supplied separately even if they're
 * included in the URL.
 * desc describes the data being fetched (for the status display).
 * The header and all CRs are removed from the data before it is returned.
 */

char *
fetch_file (const char path[], int *dlen, const char extra_headers[],
	    const char desc[], int lead_lf)
{
  char *data = 0;		/* Returned data & length */
  int length = 0, alloc = 0;
  char *auth = 0;		/* Auth string */
  int tried = 0;		/* No. of times auth string tried (<2 OK) */
  int socket;			/* Socket handle */
  char *p, *q;			/* Misc */
  char *url;                    /* The complete URL */
  char buffer[4096];		/* Read buffer */
  int config = 0;		/* For code 301: replace URL */

  url = build_url (fetcher.server, fetcher.port, path);
  if (!url)
  {
    report_error ("Out of memory");
    return 0;
  }

  /* Loop until we've fetched the data or the fetch is aborted. */
  for (;;)
  {
    int i_use_proxy = !proxy.use; /* this'll be inverted again shortly */
    int i;
    static int got_http, got_length;
    proxy.bypass = 0;

    /* Connect, trying the proxy (if configured) first. */
    set_status_text ("Connecting...", 0);
    do
    {
      i_use_proxy = !i_use_proxy;
      if (!i_use_proxy && proxy.use)
	proxy.bypass = proxy_bypass_msg;

      socket = socket_open (fetcher.server, fetcher.port, i_use_proxy);
      if (socket <= -1)
      {
	if (socket == -1)
	  status_errno ("Connect failure: ");
	goto aborted;
      }

      while ((i = socket_connected (socket)) < 0)
	if (fetch_poll ())
	{
	  socket_close (socket);
	  if (fetch.aborted)
	    goto aborted;
	  exit (0);
	}
    }
    while (i && i_use_proxy);

    if (i > 0)
    {
      errno = i;
      status_errno ("Connect failure: ");
      goto aborted;
    }

    set_status_text ("Sending request...", proxy.bypass, 0);

    /* If we're using the proxy and have enough information (from the user
     * and from reply headers), generate the authentication header.
     */
    if (!proxy.bypass && proxy.use && !auth)
    {
      auth = generate_auth (proxy.auth_type, 0, url, 0);
      if (!auth)
      {
	report_error ("Out of memory");
	goto aborted;
      }
      if (!auth[0])
	auth = 0;
    }

    /* Send the request. */
    if (send_data (socket,
		   "GET ",
		   (!proxy.bypass && proxy.use) ? url : strchr (url + 7, '/'),
		   " HTTP/1.0\r\n"
		   "Connection: close\r\n"
		   "Host: ", fetcher.server, "\r\n"
		   "Accept: \r\n"
		   "Accept-Encoding: identity\r\n",
		   (auth && !proxy.bypass) ? auth : "",
		   extra_headers,
		   "User-Agent: News-Ticker/", version, " (", osversion,
		   ")\r\n\r\n", 0))
      goto aborted;
    if (auth)
      tried++;

    set_status_text ("Fetching ", desc, " text...", proxy.bypass, 0);
    got_http = got_length = 0;

    /* Now wait for and read the response. */
    while ((i = recv (socket, buffer, sizeof (buffer), 0)) != 0)
    {
      if (i > 0)
      {
        if (length + i + 1 > alloc)
          data = realloc (data, alloc = length + i + 1);
	if (!data)
	{
	  status_error ("Out of memory");
	  goto aborted;
	}
	memcpy (data + length, buffer, i);
	length += i;
      }
      else if (errno != EWOULDBLOCK)
      {
	status_errno ("Socket read error: ");
	goto aborted;
      }
      if (fetch_poll ())
      {
	if (fetch.aborted)
	  goto aborted;
	socket_close (socket);
	exit (0);
      }
      if (!got_http && data)
      {
	data[length] = 0;
	p = strchr (data, '\n');
	if (!p)
	  continue;
	p = memchr (data, ' ', p - data);
	if (!p || memcmp (data, "HTTP/", 5) || *p != ' ')
	  goto fail;
	got_http = 1;
      }
      if (got_http && !got_length && data)
      {
	data[length] = 0;
	p = strstr (data, "\n\n");
	if (!p) p = strstr (data, "\r\n\r\n");
	if (!p)
	  continue;
	got_length = 1;
	q = stristr (data, "\nContent-Length:");
	if (q) q = strpbrk (q, " \t");
	if (q) q += strspn (q, " \t");
	if (!q)
	  continue;
	i = atoi (q);
	if (i > 4 * 1024 * 1024)
	{
	  status_error ("Ticker data is too large!");
	  goto aborted;
	}
	data = realloc (data, alloc = (p - data) + i + 7);
	if (!data)
	{
	  status_error ("Out of memory");
	  goto aborted;
	}
      }
    }
    if (!data)
    {
      set_status_text ("No data received from the ticker server.", 0);
      goto aborted;
    }
    if (!got_http)
      goto fail;

    data[length] = '\0';	/* just in case */

    p = strpbrk (data, " \t");

    /* It is; act according to the result code. */
    switch (i = atoi (p + 1))
    {
    case 200:
      /* Fetched OK. */
      socket_close (socket);
      set_status_text ("Processing ", desc, " text...", 0);
      free (auth);
      free (url);
      return process_file (data, length, dlen, lead_lf);

    case 301:
      config = find_server (fetcher.label);
      if (config < 1 || fetcher.port != servers[config].port
	  || stricmp (fetcher.server, servers[config].server)
	  || strcmp (fetcher.stories_path, servers[config].stories_path))
	config = 0;
      goto moved_common;
    case 302:
      config = 0;
      moved_common:
      p = stristr (data, "\nLocation:");
      if (p)
	p = strpbrk (p, " \t");
      if (!p)
        break;
      p += strspn (p, " \t");
      free (url);
      if (strchr (p, '\r'))
        *(strchr (p, '\r')) = '\0';
      else if (strchr (p, '\n'))
        *(strchr (p, '\n')) = '\0';
      if (*p == '/')
      {
        /* incomplete URL, same server/port */
        url = build_url (fetcher.server, fetcher.port, ++p);
        if (config && strlen (p) < 256)
        {
          strcpy (servers[config].stories_path, p);
          strcpy (fetcher.stories_path, p);
          goto update_servers_window;
        }
      }
      else
      {
        char *slash, *colon;
	if (stristr (p, "http://") != p)
	{
	  set_status_text ("Cannot fetch; redirected to a non-HTTP URL");
	  goto aborted;
	}
	slash = strchr (p + 7, '/');
	if (slash)
	  *slash++ = '\0';
	colon = strrchr (p + 7, ':');
	if (colon)
	  *colon++ = '\0';
	url = build_url (p + 7, colon ? atoi (colon) & 0xFFFF : 80,
			 slash ? slash : "");
        if (config && strlen (p + 7) < 256
	    && (!slash || strlen (slash) < 256))
	{
          servers[config].port = colon ? atoi (colon) & 0xFFFF : 80;
          if (!servers[config].port)
            servers[config].port = 80;
          strcpy (servers[config].server, p + 7);
          strcpy (servers[config].stories_path, slash ? slash : "");
          fetcher.port = servers[config].port;
          strcpy (fetcher.server, p + 7);
          strcpy (fetcher.stories_path, slash ? slash : "");
          update_servers_window:
	  servers[config].modified |= MODIFIED_AUTOMATICALLY;
          if (!strcmp (get_icon_text (wind.servers, 1), fetcher.label)
              && is_window_open (wind.servers))
            open_servers_window (0, 0, 0);
	}
      }
      if (!url)
      {
        report_error ("Out of memory");
        return 0;
      }
      set_status_text ((i == 302) ? "Temporary redirect to " :
				    "Redirecting to ",
		       url, " ...", 0);
      socket_close (socket);
      free (data);
      data = 0;
      length = alloc = 0;
      /* Try again... */
      continue;

    case 407:
      /* The proxy wants authentication details.
       * If it wants basic, ask immediately.
       * if it wants digest, try again *then* ask the user.
       */
      if (!auth || tried == 2 - (proxy.auth_type == 'b'))
      {
	set_status_text ("Proxy authentication required. (",
			 proxy.auth_type == 'b' ? "Basic)" : "Digest)", 0);
	if (task_handle)
	  get_auth_details (data, url);
	else
	  get_auth_details_singletask (data, url);
	if (fetch.aborted)
	  goto aborted;
	/* Got them. Generate the authentication header before we discard the
	 * proxy's response.
	 */
	free (auth);
	tried = 0;
      }
      auth = generate_auth (proxy.auth_type, data, url, 1);
      if (!auth)
      {
	report_error ("Out of memory");
	goto aborted;
      }
      if (!auth[0])
	auth = 0;
      socket_close (socket);
      free (data);
      data = 0;
      length = 0;
      /* Try again... */
      continue;
    }
    break;
  }

fail:
  p = memchr (data, '\n', length);
  if (!p)
    p = data + length;
  if (p > data && p[-1] == '\r')
    p[-1] = '\0';
  else
    p[0] = '\0';
  p = strchr (data, ' ');
  if (memcmp (data, "HTTP/", 5) && p)
    set_status_text ("Unexpected response: ", data, 0);
  else
    set_status_text (p + 1, 0);

aborted:
  socket_close (socket);
  free (data);
  free (auth);
  free (url);
  return 0;
}


/* Free the old data. We don't want it around when we're fetching new data */

void
free_old_data (void)
{
  if (ticker_data.headings)
    free (ticker_data.headings);

  if (ticker_data.stories)
  {
    int i;
    for (i = ticker_data.num_headings; i; free (ticker_data.stories[--i]));
    free (ticker_data.stories);
  }

  if (fetcher.free_old_data)
    fetcher.free_old_data ();
  free (headings_data);
  free (stories_data);
  free_resources_menu ();

  ticker_data.headings = 0;
  ticker_data.stories = 0;
  headings_data = 0;
  stories_data = 0;
  ticker_data.num_headings = fetcher.num_headings;
}


/* Claim initial storage space for story data. */

int
init_stories (void)
{
  int i;
  ticker_data.stories =
    calloc (ticker_data.num_headings, sizeof (struct story *));
  if (!ticker_data.stories)
  {
    status_error ("Out of memory");
    free (stories_data);
    stories_data = 0;
    return 0;
  }
  for (i = ticker_data.num_headings;
       i;
       ticker_data.stories[--i] = 0)
    ;
  return 1;
}


/* Common story fetch code. */

int
fetch_stories_file (int lead_lf)
{
  char cache_control[32];
  int length;

  if (fetch.aborted)
    return 0;
  fetch.aborted = 0;

  sprintf (cache_control, "Cache-Control: max-age=%i\r\n",
	   opt.update_every * 60);
  free (stories_data);
  stories_data =
    fetch_file (fetcher.stories_path, &length, cache_control, "stories",
		lead_lf);
  return length;
}


/* Free story storage space. */

void
free_stories (void)
{
  int i;
  free (stories_data);
  stories_data = 0;
  for (i = ticker_data.num_headings; i; free (ticker_data.stories[--i]))
    ;
  free (ticker_data.stories);
  ticker_data.stories = 0;
}


/* Add a headline in the appropriate section.
 * The new entry points to text near ptr; it isn't duplicated.
 * (We also go redirector killing in here...)
 */

char *
append_story (int num, char *ptr)
{
  int i;
  char *p;

  if (!fetcher.find_last_update (ptr))
    num = 0;

  if (num >= ticker_data.num_headings)
  {
    /* Eek! Extra topics! Something's broken at the Beeb! */
    if (num > 31)
      num = 31;			/* Hmm... hard limit... */
    i = ticker_data.num_headings;
    ticker_data.num_headings = num + 1;
    if (ticker_data.headings)
    {
      ticker_data.headings =
	realloc (ticker_data.headings,
		 ticker_data.num_headings * sizeof (const char *));
      if (!ticker_data.headings)
      {
	status_error ("Out of memory");
	return 0;
      }
      memset (ticker_data.headings + i, 0,
	      (ticker_data.num_headings - i) * sizeof (const char *));
    }
    ticker_data.stories =
      realloc (ticker_data.stories,
	       ticker_data.num_headings * sizeof (struct story *));
    if (!ticker_data.stories)
    {
      status_error ("Out of memory");
      return 0;
    }
    for (; i < ticker_data.num_headings; ticker_data.stories[i++] = 0);
  }

  p = fetcher.find_story (&ptr);
  if (!p)
    return fetcher.find_next_item (ptr);

  i = 0;
  if (ticker_data.stories[num])
    while (ticker_data.stories[num][i].headline)
      i++;

  ticker_data.stories[num] = realloc (ticker_data.stories[num],
				       ((i + 9) & ~7) * sizeof (struct story));
  if (!ticker_data.stories)
  {
    status_error ("Out of memory");
    return 0;
  }

  ticker_data.stories[num][i].headline = p;
  ticker_data.stories[num][i + 1].headline = 0;
  ticker_data.stories[num][i].description =
    fetcher.find_description ? fetcher.find_description (&ptr) : 0;
  ticker_data.stories[num][i].publish_date =
    fetcher.find_publish_date ? fetcher.find_publish_date (&ptr) : 0;

  p = fetcher.find_url (&ptr);
  ticker_data.stories[num][i].url = (p && *p <= ' ') ? 0 : p;

  return fetcher.find_next_item (ptr);
}


static void
free_resources_menu (void)
{
  if (resources_menu && resources_menu->width)
  {
    struct menu_entry *item = &resources_menu->items[0] - 1;
    do
    {
      if (item->icon_flags & 256)
        free (item->text.it.text);
    } while (!((item++)->menuflags & 128));
    free (resources_menu);
    resources_menu = 0;
  }
}


/* Create the URLs menu */

static void
create_resources_menu (void)
{
  if (fetcher.fetch_resources == 0)
    return;

  free_resources_menu ();
  resources_menu = malloc (28 + sizeof (struct menu_entry));

  if (!resources_menu)
  {
    report_error ("Out of memory");
    return;
  }

  strcpy (resources_menu->title, "Go to");
  resources_menu->colours = 0x70207;
  resources_menu->width = 0;
  resources_menu->height = 44;
  resources_menu->gap = 0;

  if (!fetcher.fetch_resources () || !resources_menu || !resources_menu->width)
    free_resources_menu ();
}


char *
append_resource (char *ptr)
{
  base_menu_type *newmenu;
  struct menu_entry *item = &resources_menu->items[0];
  int i;
  char *p, *u;

  p = fetcher.find_resource (&ptr);
  if (!p)
    return fetcher.find_next_item (ptr);
  u = fetcher.find_url (&ptr);
  if (!u)
    return fetcher.find_next_item (ptr);

  i = 0;
  if (resources_menu->width)
    while ((item[i++].menuflags & 128) == 0)
      ;

  newmenu = malloc (28 + 24 + i * 24);
  if (!newmenu)
  {
    status_error ("Out of memory");
    return 0;
  }

  memcpy (newmenu, resources_menu, 28 + i * 24);
  free (resources_menu);
  resources_menu = newmenu;
  item = &(newmenu->items[0]);
  if (newmenu->width)
    item[i - 1].menuflags &= ~128;
  item[i].menuflags = 128;
  item[i].submenu = -1;
  item[i].icon_flags = 0x07000121;
  item[i].text.it.text = malloc (strlen (p) + strlen (u) + 2);
  if (!item[i].text.it.text)
  {
    status_error ("Out of memory");
    return 0;
  }
  sprintf (item[i].text.it.text, "%s%c%s", p, 0, u);
  item[i].text.it.validation = "";
  item[i].text.it.text_len = strlen (item[i].text.it.text);
  if (newmenu->width < item[i].text.it.text_len * 16 + 12)
    newmenu->width = item[i].text.it.text_len * 16 + 12;

  return fetcher.find_next_item (ptr);
}


/* Fetch all of the ticker information, and set up the ticker window. */

int
fetch_ticker (int force)
{
  static int lock = 0;		/* just in case :-) */

  static int inited = 0;
  if (!inited)
  {
    atexit (fetch_shutdown);
    inited = 1;
  }

  if (ticker_data.state == state_LOADING)
    return 0;

  if (force)
    fetch.allowed = 1;
  if (!is_fetch_allowed ())
  {
    set_status (state_NOT_CONNECTED);
    open_ticker_window (NULL, opt.update_to_front ? -1 : 0, 1);
    return 0;
  }

  assert (lock == 0);
  lock = 1;
  fetch.aborted = 0;

  set_status (state_LOADING);
  open_ticker_window (NULL, opt.update_to_front ? -1 : 0, 1);
  free_old_data ();

  if (stricmp (proxy.server, opt.proxy))
    cache.pass = cache.user = 0;

  strcpy (proxy.server, opt.proxy);
  proxy.port = opt.proxyport;
  proxy.use = opt.use_proxy;

  if (testmode || (fetcher.fetch_stories () && fetcher.fetch_headings ()))
    set_status (state_ACTIVE);
  else
  {
    set_status (state_UNAVAILABLE);
    free_old_data ();
  }

  if (task_handle)
  {
    process_stories ();
    if (fetch.aborted)
      set_status_text ("Fetch aborted.", 0);
    else
      create_resources_menu ();
    open_ticker_window (NULL, opt.update_to_front ? -1 : 0, 1);
    if (ticker_data.update_time)
    {
      long now = read_monotonic_time();
      while (now - ticker_data.last_update <= -ticker_data.update_time)
	ticker_data.last_update += ticker_data.update_time;
    }
    else
      ticker_data.last_update = read_monotonic_time ();
  }

  lock = 0;
  return !fetch.aborted;
}


/* Flag an aborted fetch */

void
abort_fetch (void)
{
  fetch.aborted = 1;
}


int
is_fetch_allowed (void)
{
  if (*opt.connect_var)
    fetch.allowed = getenv (opt.connect_var) != 0;
  else
    fetch.allowed = 1;
  return fetch.allowed;
}


/* Proxy authorisation window */

void
click_proxy_auth (void)
{
  if (poll_block.mouse_click.buttons != 2)
    switch (poll_block.mouse_click.icon_handle)
    {
    case 7:
      fetch.aborted = 1;
      break;
    case 8:
      wimp_close_window (&wind.proxy_auth);
      break;
    }
}


int
key_proxy_auth (void)
{
  switch (poll_block.key_pressed.code)
  {
  case 13:
    if (poll_block.key_pressed.icon_handle == 6)
    {
      wimp_close_window (&wind.proxy_auth);
      return 1;
    }
    break;
  case 27:
    return fetch.aborted = 1;
  }

  return 0;
}


/* Do we need to update the ticker yet?
 * (We *don't* check the options here.)
 */

#define get_update_period()\
  ((ticker_data.update_time > opt.update_every * 6000L)\
   ? ticker_data.update_time\
   : opt.update_every * 6000L)


int
time_to_update (void)
{
  return read_monotonic_time () - ticker_data.last_update >=
    get_update_period ();
}


int
get_next_update_time (void)
{
  return ticker_data.last_update + get_update_period ();
}
