
/***************************************************************************
 *   Copyright (C) 1997 to 2004 by Jonathan Duddington                     *
 *   email: jonsd@users.sourceforge.net                                    *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 3 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write see:                           *
 *               <http://www.gnu.org/licenses/>.                           *
 ***************************************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

#include "menu.h"
#include "wimp.h"
#include "werr.h"
#include "dbox.h"
#include "os.h"
#include "flex.h"
#include "msgs.h"

#include "narc.h"
#include "hdrs.h"


#ifdef MemCheck
#include "MemCheck:Flex.h"
#endif

extern char *sort_card_file;


extern ART_FILE_HEADER art_file_header[N_ART_FILES];
extern unsigned int *ixlist_article;     /* card sorted by article address */
extern char *string_repair;
extern char compact_attempted[N_ART_FILES];
extern char appended_art_file[N_ART_FILES];   /* note which article files have received news */
extern int  error_param;

extern OPTIONS options;
extern FOLDREC list_fr[N_LISTS];
extern CARDFILE_REC cardfile[N_CARDFILES];
extern BOX box_table[N_BOXES];
extern char box_selected[N_BOXES];
extern TEXTR *text_view;
extern char errmess_article_read[];
extern int cardfile_changed[];
extern int dont_save_cardfile;

static FILE *f_errors=NULL;
static char *fname_errors = "<Pluto$Dir>.tmp.error_log";
static int  error_log_count = 0;



void error_log_open()
/*******************/
{
	if((f_errors = fopen_werr(fname_errors,"w","error log")) != NULL)
	{
		fprintf(f_errors,"Pluto error log\n\n");
	}

	error_log_count = 0;
}   /* end of error_log+open */


void error_log_report(char *string)
/*********************************/
{
	if(f_errors != NULL)
	{
		fclose(f_errors);

		if(error_log_count > 0)
		{
			werr(0,"%d %s %s",error_log_count,string,string_repair);
			dataopen(fname_errors,0xfff);
		}
	}
	error_log_count = 0;
}   /* end of error_log_report */




void write_log_message(char *string)
/**********************************/
{
	FILE *f;
	char buf[40];

	f = fopen_pluto("Backup.log","a");
	if(f != NULL)
	{
		while(*string=='\n')
			fprintf(f,"%c",*string++);  /* put initial NL before the date */

		get_today_date2(buf);
		fprintf(f,"%s  %s\n",buf,string);
		fclose(f);
	}
}   /* end of write_log_message */



int article_file_size(int cf,int fileno)
/**************************************/
{
	char fname[128];

	sprintf(fname,"%s.Articles%d",cardfile[cf].path,fileno);
	return(get_filelength(fname));
}  /* end of article_file_size */




int article_file_total_size(int cf)
/*********************************/
{
	int  i;
	int  total=0;
	int  n_art_files;

	if(cf==0)
		n_art_files = N_ART_FILES;
	else
		n_art_files = N_ART_FILES_X;

	for(i=0; i<n_art_files; i++)
	{
		total += article_file_size(cf,i);
	}
	return(total);
}   /* end of article_file_total_size */




int article_file_recover(int cf, int reason)
/******************************************/
/* Create a new card file from information in the article files */
{
	FILE *f;
	FILE *f_card;
	static int  fileno;
	int  file_length;
	int  displ;
	int  start;
	int  cardsize;
	int  word;
	int  length;
	int  err_displ;
	int  ok_displ;
	int  error_fileno = -1;
	int  n_art_files;
	ART_FILE_HEADER *art_file_hdr;

	int  total;
	int  acc;
	char *reason_string;
	int  box;
	int  external_box;
	int  error;

	int  filebuf_size;
	char *filebuf;
	char *fp;

	dbox d_err = 0;
	CARD card;
	ART_HEAD art_head;
	CARD_HEADER card_header;

	char fname[256];

	d_err = dbox_new("Message");
	dbox_hidefield(d_err,0);

	if(cf==0)
		n_art_files = N_ART_FILES;
	else
		n_art_files = N_ART_FILES_X;

	art_file_hdr = &art_file_header[cf_filenum(cf,0)];

	if(reason==1)
		reason_string = "missing";
	else
		reason_string = "out of date";

	if(reason != 0)
	{
		if(cf==0)
		{
			sprintf(fname,"Article index is %s, will re-make it (%d).  Perhaps Pluto didn't quit properly when last used.",reason_string,reason);

			if(query2(fname,"Continue","Quit")==0)
				exit(3);
		}
		else
		{
			sprintf(fname,"External box index is %s, will re-make it (%d).",reason_string,reason);

			if(query2(fname,"Continue","Abort")==0)
				return(-1);
		}
	}

	strcpy(fname,"Now rebuilding index");

	dbox_setfield(d_err,1,fname);
	dbox_show(d_err);

	call_event_process(10);

	/* create new article headers file */
	memset(art_file_hdr,0,sizeof(ART_FILE_HEADER)*n_art_files);
	for(fileno=0; fileno<n_art_files; fileno++)
	{
		art_file_hdr[fileno].version = ART_FILE_VERSION;
	}

	/* create card file and write header */
	sprintf(fname,"%s.cards",cardfile[cf].path);

	error = 0;
	if(get_filelength(fname) > 0)
	{
		if(remove(fname) != 0)
			error = 1;
	}
	if(error == 0)
	{
		if((f_card = fopen(fname,"w+")) == NULL)
			error = 2;
	}
	if(error)
	{
		dbox_dispose(&d_err);
		werr(0,"Can't update index file. Write protected? '%s'",fname);
		if(cf==0)
			exit(3);
		else
			return(-1);
	}

	memset(&card_header,0,sizeof(card_header));
	card_header.version = CARD_FILE_VERSION;
	fwrite(&card_header,1,sizeof(card_header),f_card);

	error_log_open();
	visdelay2_begin();
	set_lock_lists(1,NULL);

	/* process each article file in turn */

	total = article_file_total_size(cf) / 256;
	acc = 0;

	external_box = cf_to_box(cf);

	filebuf_size = 0x40000;    /* 256k buffer for card file */
	filebuf = malloc(filebuf_size);
	fp = filebuf;  /* may be NULL if malloc failed */

	for(fileno=0; fileno<n_art_files; fileno++)
	{
		if(total==0) total=1;
		visdelay2_percent(acc*100/total);

		f = article_fopen(cf,fileno,"r");
		if(f == NULL)
		{
			/* article file does not exist, create it */
			f = article_fopen(cf,fileno,"w");
			fwrite(&art_file_hdr[fileno],1,sizeof(ART_FILE_HEADER),f);
			fclose(f);
			continue;
		}

		/* find length of article file */
		file_length = article_file_size(cf,fileno);
		acc += (file_length / 256);

		art_file_hdr[fileno].size = file_length;

		fseek(f,sizeof(ART_FILE_HEADER),SEEK_SET);  /* skip file header */

		while(!feof(f))
		{
			call_event_process(0);

			displ = (int)ftell(f);

			if(fread(&art_head,sizeof(ART_HEAD),1,f)==0) break;
			if(feof(f)) break;

			if(art_head.magic != MAGIC)
			{
				/* skip to look for start of article marker */
				err_displ = displ;
				ok_displ = 0;

				displ = (displ + 3) & ~3;   /* next word boundary */
				fseek(f,displ,SEEK_SET);
				while(!feof(f))
				{
					displ = (int)ftell(f);
					if(fread(&word,4,1,f)==0) break;

					if(word == MAGIC)
					{
						fseek(f,displ,SEEK_SET);
						fread(&art_head,sizeof(ART_HEAD),1,f);
						ok_displ = displ;
						break;
					}
				}
				if(f_errors != NULL)
				{
					if((error_fileno != -1) && (error_fileno != fileno))
						fputc('\n',f_errors);
					error_fileno = fileno;
					fprintf(f_errors,"Error in Article File %2d at %6x continue at %6x\n",fileno,err_displ,ok_displ);
				}
				error_log_count++;
				if(feof(f)) break;
			}

			/* top bit set in 'length' means that article has been deleted */
#ifdef deleted
			if(art_head.length < 0)
			{
				/* to recover deleted articles in an emergency */
				art_head.length &= 0x7fffffff;
			}
#endif

			if(art_head.length < 0)
			{
				art_file_hdr[fileno].free_space += (art_head.length & ART_LENGTH_MASK);
			}
			else
			{
				art_file_hdr[fileno].n_articles++;

				start = (int)ftell(f);           /* read card information */
				fread(&card,1,sizeof(CARD),f);
				cardsize = card_size(&card);

				length = ((card.alength & ART_LENGTH_MASK)+ 3) & ~3;
				if(length < art_head.length)
				{
					art_file_hdr[fileno].free_space += (art_head.length - length);
				}

				/* write out card */
				card.addr = (fileno << 24) + displ;
				if((card.status & STATUS_MASK) <= STATUS_NEW)
				{
					card.status += (STATUS_UNREAD-STATUS_NEW);
				}

				if(cf > 0)
				{
					card.date_box = (card.date_box & ~BOX_MASK) + (external_box << 26);
				}
				else
				{
					/* check that the Box exists */
					box = card.date_box >> 26;
					if((box < BOX_EXTERN) && (box_table[box].name[0]==0))
					{
						/* no, so create the box for this article */
						memset(&box_table[box],0,sizeof(BOX));
						sprintf(box_table[box].name,"LOST+FOUND%d",box);
						save_boxes_file();
					}
				}

				if(fp == NULL)
				{
					/* no buffer, write out the individual card */
					fwrite(&card,1,cardsize,f_card);

					/* pad to next word boundary */
					start = (int)ftell(f_card);
					while((start & 3) != 0)
					{
						fputc(0,f_card);
						start++;
					}
				}
				else
				{
					if((&filebuf[filebuf_size] - fp) < sizeof(CARD))
					{
						/* buffer full, flush it and reset */
						fwrite(filebuf,1,fp-filebuf,f_card);
						fp = filebuf;
					}
					memcpy(fp,&card,cardsize);
					fp += cardsize;

					/* pad to next word boundary */
					while((((int)fp) & 3) != 0)
					{
						*fp++ = 0;
					}
				}

				card_header.n_cards++;
			}

			/* move to next article */
			displ = displ + (art_head.length & ART_LENGTH_MASK) + sizeof(ART_HEAD);
			fseek(f,displ,SEEK_SET);
		}

		fclose(f);
	}

	if(fp != NULL)
	{
		fwrite(filebuf,1,fp-filebuf,f_card);
	}

	clear_lock_lists(1);
	visdelay2_end();

	/* write out card_header */
	rewind(f_card);
	fwrite(&card_header,1,sizeof(card_header),f_card);
	fclose(f_card);

	article_save_file_headers(cf);
	box_bin_hide();

	if(filebuf != NULL)
	{
		free(filebuf);
	}

	if(d_err != 0)
	{
		dbox_dispose(&d_err);
	}
	cardfile_changed[cf] = 0;

	error_log_report("errors found in articles files");

	return(0);
}   /* end of article_file_recover */






static void save_short_header(FILE *f, CARD_EXPANDED *cardex, int outgoing, int control)
/**************************************************************************************/
/* Control: 1= used in export full header */
/* This is a copy of the same procedure in textextra module */
{
	char *p;
	char *direction;
	char *user_label;
	char *re;
	OPTIONS_MAILBOX *mbox;
	char buf[128];

	if(outgoing & STATUS_BIT_OG)
	{
		user_label = "From";
		if(outgoing & STATUS_BIT_NEWS)
			direction = "Newsgroups";
		else
			direction = "To";
	}
	else
	{
		direction = "From";
		user_label = "To";
	}

	strcpy(buf,cardex->author);
	if(strchr(cardex->author,'@')==NULL)
	{
		p = addr_lookup(cardex->author);
		if(p != NULL)
		{
			sprintf(buf,"%s <%s>",cardex->author,p);
		}
	}
	fprintf(f,"%s: %s\n",direction,buf);

	if((control == 0) && (cardex->user > 0))
	{
		mbox = &options.mailbox[cardex->user-1];
		fprintf(f,"%s: %s\n",user_label,mbox->email_addr);
	}

	if(cardex->reply)
		re = "Re: ";
	else
		re = "";
	fprintf(f,"Subject: %s%s\n",re,cardex->title);

	if(control==0)
		fprintf(f,"Date: %s\n",decode_date(cardex->date,4));

	if(control == 0)
	{
		if((outgoing & STATUS_BIT_OG) && (outgoing & STATUS_BIT_NEWS))
		{
		}
		else
		{
			if(cardex->source > 0)
			{
				if(cardex->status_other & STATUS_BIT_NEWS)
					p = "Newsgroups";
				else
					p = "Source";

				fprintf(f,"%s: %s\n",p,expand_source(cardex->source,0));
			}
		}
	}

	if(cardex->cats[0] != 0)
	{
		fprintf(f,"X-Categories: %s\n",cardex->cats);   /* ???? */
	}
	if(cardex->keywords[0] != 0)
	{
		fprintf(f,"Keywords: %s\n",cardex->keywords);
	}

	if(control == 0)
		fputc('\n',f);
}   /* end of save_short_header */





void article_files_search_fault(int start)
/****************************************/
/* Read all the articles, writing details of each to a file, to
   find which one causes a crash */
{
	FOLDREC *fr;
	FILE *f_out;
	int  cardsize;
	int  length;
	int  error;
	CARD *cptr;
	int  fileno;
	int  fileno_start;
	int  fileno_end;
	dbox d;
	CARD_EXPANDED card_expanded;
	char path[40];
	char report_string[128];

	static unsigned int *ixlist;
	static int ix=0;
	static int last_fileno=0;
	static int end_fileno = 64;
	static int n_entries;
	static TEXTR *text_temp;
	static int in_progress = 0;
	static int save_each_article = 0;


	if(start == 1)
	{
		if(in_progress)
		{
			ix = 0x7fffffff;  /* force termination */
			in_progress = 0;
			return;
		}

		d = dbox_new("FindFault");
		dbox_show(d);
		if(dbox_fillin(d) == 0)
		{
			fileno_start = dbox_getnumeric(d,3);
			fileno_end = dbox_getnumeric(d,4);
			save_each_article = dbox_getnumeric(d,5);
			dbox_dispose(&d);
		}
		else
		{
			dbox_dispose(&d);
			return;
		}


		text_temp = calloc(1,sizeof(TEXTR));
		if(text_temp == NULL)
			return;

		fr = &list_fr[0];

		visdelay2_begin();

		/* sort by article file address */
		if(article_make_index(fr,&ixlist)==NULL)
		{
			visdelay2_end();
			return;
		}

		ix = 0;
		n_entries = fr->n_entries;
		end_fileno = fileno_end + 1;
		if(fileno_start > 0)
		{
			while(ix < n_entries)
			{
				cptr = (CARD *)((ixlist[ix++] & IXLIST_MASK) << 2);

				if(((cptr->addr >> 24) & 0xff) >= fileno_start)
					break;
			}
		}

		f_errors = fopen_werr_pluto("tmp.SearchLog","w",NULL);
		if(f_errors!=NULL)
			fclose(f_errors);
		null_events(8,1);
		in_progress = 1;
	}

	while(ix < list_fr[0].n_entries)
	{
		cptr = (CARD *)((ixlist[ix] & IXLIST_MASK) << 2);

		fileno = (cptr->addr >> 24) & 0xff;
		if(fileno > end_fileno)
			break;

		if(fileno != last_fileno)
		{
			f_errors = fopen_pluto("tmp.SearchLog","a");
			if(f_errors != NULL)
			{
				fprintf(f_errors,"------ Start article file %d ------\n\n",fileno);
				fclose(f_errors);
			}
		}
		last_fileno = fileno;


		unpack_card(cptr,&card_expanded);
		sprintf(report_string,"Box: %s\nAddress: File %d at &%x\n",
				box_table[card_expanded.docbox].name,card_expanded.addr >> 24,card_expanded.addr & 0xffffff);


		if(save_each_article != 0)
		{
			f_out = fopen_pluto("tmp.Article","w");
			if(f_out != NULL)
			{
				save_short_header(f_out,&card_expanded,cptr->status & STATUS_BIT_OG,0);
				fprintf(f_out,"%s",report_string);
				fclose(f_out);
			}
		}

		cardsize = card_size(cptr);
		length = cptr->alength & ART_LENGTH_MASK;

		length = length - cardsize;    /* length of text (internet header + text body) */
		if(length <= 0)
			continue;

		visdelay2_percent(ix*100/n_entries);

		error = article_read(text_temp, cptr->addr, cptr->date_box >> 26, cardsize, length, 1, 0, 1);

		if(error != 0)
		{
			f_errors = fopen_pluto("tmp.SearchLog","a");
			if(f_errors != NULL)
			{
				fprintf(f_errors,"----------\n");
				save_short_header(f_errors,&card_expanded,cptr->status & STATUS_BIT_OG,0);
				fprintf(f_errors,"%s",report_string);
				fprintf(f_errors,"%s\n\n\n",errmess_article_read);
				fclose(f_errors);
			}
		}

		if(text_temp->text_base != NULL)
		{
			flex_free((flex_ptr)&text_temp->text_base);
			text_temp->text_buf_size = 0;
		}

		ix++;
		if(((ix & 0x3)==0) || (length > 40000))
			return;
	}

	null_events(8,0);
	visdelay2_end();


	sprintf(path,"%stmp.Article",pluto_path);
	remove(path);  /* we haven't crashed */
	flex_free((flex_ptr)&ixlist);
	free(text_temp);
	in_progress = 0;
	announce_message(msgs_lookup("Y10"),1);   // 'search fault' completed
}   /* end of search_fault */






void article_read_references(CARD *cptr, TEXTR *t)
/************************************************/
{
	int  argument2;
	int  arg_end;
	int  key;
	char *p;
	int  msgid;
	int  start_pos;
	int  length;
	int  cardsize;
	ARTICLE_IN art_in;
	static CARD_EXPANDED *card_ex1 = NULL;

	if((cptr->user & INET_HDR_MASK) == 0)
		return;

	cardsize = card_size(cptr);
	length = cptr->alength & ART_LENGTH_MASK;
	length = length - cardsize;    /* length of text (internet header + text body */

	if(article_read(t, cptr->addr, cptr->date_box >> 26, cardsize, length, 1, 0, 1) != 0)
		return;

	if(card_ex1 == NULL)
	{
		card_ex1 = malloc(sizeof(CARD_EXPANDED));
		if(card_ex1 == NULL)
			return;
	}
	unpack_card(cptr,card_ex1);

	msgid = 0;
	start_pos = 0;

	art_in.in_reply_to = -1;
	art_in.references = -1;

	while((key = interpret_article_key(&t->text_base,&start_pos,t->text_length,&argument2,&arg_end)) >= 0)
	{
		switch(key)
		{
		case 8:   /* References */
			art_in.references = argument2;
			art_in.references_end = arg_end;
			break;

		case 29:  /* in reply to */
			art_in.in_reply_to = argument2;
			art_in.in_reply_to_end = arg_end;
			break;

		case 9:   /* Message-ID */
			p = get_reference(&t->text_base,argument2,arg_end,1);
			if(p != NULL)
			{
				msgid = gethash32(p);
			}
			break;
		}
	}

	card_ex1->n_refs = 0;
	card_ex1->parent = 0;
	card_ex1->references[0] = 0;
	interpret_references(card_ex1,&t->text_base,&art_in);

	if(msgid != 0)
		card_ex1->msgid = msgid;

	pack_card(cptr,card_ex1);
}   /* end of article_read_references */




int article_file_set_references(void)
/***********************************/
/* Go through an article file and recover waste space.  Compact the
   data and update the pointers in the card file.
   Need to make article_ixlist first */
{
	FILE *f;
	FILE *f_out;
	int  fileno;
	int  displ;
	int new_displ;
	int seek_displ;
	int prev_displ;
	int key;
	int key2;
	int  ix;
	int  length;
	int  head_length;
	CARD *cptr;
	CARD *prev_cptr=NULL;
	CARD *cptr_trace = NULL;
	int  errors=0;
	int  box;
	int  cardsize1;
	int  cardsize2;
	char *p;
	int  total;
	int  acc=0;
	TEXTR *text_temp;
	ART_HEAD art_head;
	ART_FILE_HEADER file_header;
	CARD card;
	char fname[128];
	char fname2[128];

	int filebuf_size;
	char *filebuf=NULL;
	char *fp;

	if(query("This may take a long time.  The maximum number of references kept for each message is set by Preferences->News/Mail->Keep_references",NULL)==0)
		return(0);


	/* close any external boxes */
	box_close_all_external();

	if(article_make_index(&list_fr[0],&ixlist_article) == NULL)
	{
		return(0);
	}

	text_temp = calloc(1,sizeof(TEXTR));
	if(text_temp == NULL)
		return(0);

	filebuf_size = 0x20000;
	fp = filebuf = malloc(filebuf_size);
	if(filebuf==NULL)
	{
		malloc_err(55);
		free(text_temp);
		return(0);
	}

	set_lock_lists(1,NULL);

	visdelay2_begin();
	total = article_file_total_size(0) / 256;


	for(fileno=0; fileno<N_ART_FILES; fileno++)
	{
		visdelay2_percent(acc*100/total);
		acc += (article_file_size(0,fileno) / 256);

		sprintf(fname,"%s.articles%d",pluto_articles,fileno);
		sprintf(fname2,"%s.articles98",pluto_articles);
		file_copy(fname,fname2);

		if((f = fopen_werr(fname2,"r",NULL)) == NULL) return(0);
		sprintf(fname,"%s.articles99",pluto_articles);
		if((f_out = fopen_werr(fname,"w",NULL)) == NULL)
		{
			fclose(f);
			return(0);
		}


		art_file_header[fileno].n_articles = 0;

		memset(&file_header,0,sizeof(file_header));
		file_header.version = ART_FILE_VERSION;
		fread(&file_header,sizeof(file_header),1,f);
		fwrite(&file_header,sizeof(file_header),1,f_out);
		displ = (int)ftell(f);

		/* ixlist_article is sorted by article address */
		key = (fileno << 24) + displ;
		key2 = (fileno + 1) << 24;

		/* find the start of ixlist entries for the specified article file */
		for(ix=0; ix < list_fr[0].n_entries; ix++)
		{
			cptr = (CARD *)((ixlist_article[ix] & IXLIST_MASK) << 2);
			if(cptr->addr >= key)
				break;
		}

		if((ix < list_fr[0].n_entries) || (cptr->addr < key2))
		{
			while(!feof(f))
			{

				if((ix & 7) == 0)
				{
					call_event_process(0);

					if((ix & 0xff) == 0)
						visdelay2_percent(acc*100/total);
				}

				displ = (int)ftell(f);
				if(fread(&art_head,sizeof(art_head),1,f)==0) break;
				if(feof(f))
					break;

				head_length = art_head.length & ART_LENGTH_MASK;
				if(art_head.magic != MAGIC)
				{
					/* Article file is corrupt */
					seek_displ = displ & ~4;

					for(;;)
					{
						seek_displ += 4;
						fseek(f,seek_displ,SEEK_SET);
						if(fread(&art_head,sizeof(art_head),1,f)==0) break;
						if(art_head.magic == MAGIC)
						{
							break;
						}
					}
					errors=2;
					break;
				}

				prev_cptr = cptr;

				key = (fileno << 24) + displ;
				while((ix < list_fr[0].n_entries) && (cptr->addr < key))
				{
					cptr_trace = cptr;  /* just for testing */
					cptr = (CARD *)((ixlist_article[++ix] & IXLIST_MASK) << 2);
				}

				if((ix >= list_fr[0].n_entries) || (cptr->addr > key))
				{
					/* Card file and Article file don't match */
					if(art_head.length >= 0)
					{
						fread(&card,1,16,f);
						box = card.date_box >> 26;
						if(box < BOX_BIN)
						{
							errors=1;
							break;
						}
					}
				}
				else if(art_head.length < 0)
				{
					/* mark this record as deleted, it will be removed from the
					   cardfile when it is saved when the program terminates */
					cptr->date_box = (cptr->date_box & ~BOX_MASK) + ((unsigned)BOX_DELETED << 26);
				}
				else
				{
					/* this block is not garbage, copy to new file */

					new_displ = (int)ftell(f_out);
					if(fp != NULL)
					{
						new_displ += (fp - filebuf);
					}

					fread(&card,1,22,f);
					cardsize1 = card_size(&card);
					p = (char *)(&card);
					fread(p+22,1,cardsize1-22,f);

					card.addr = (fileno << 24) + displ;
					length = card.alength & ART_LENGTH_MASK;
					length -= cardsize1;

					/* set the message-id and references info. in the card */
					article_read_references(&card,text_temp);

					cardsize2 = card_size(&card);

					length += cardsize2;
					length = (length + 3) & ~3;   /* round up to word boundary */
					art_head.length = length;

					/* don't need to update the card file, we'll
					   rebuild it later  */
					card.addr = (cptr->addr & ~ART_ADDR_MASK) + new_displ;
					/*   cptr->addr = card.addr; */

					if((&filebuf[filebuf_size]-fp) < (sizeof(art_head)+cardsize2))
					{
						fwrite(filebuf,1,fp-filebuf,f_out);
						fp = filebuf;
					}
					memcpy(fp,&art_head,sizeof(art_head));
					fp += sizeof(art_head);
					memcpy(fp,&card,cardsize2);
					fp += cardsize2;

					length -= cardsize2;

					while(length >= (&filebuf[filebuf_size]-fp))
					{
						fread(fp,1,(&filebuf[filebuf_size]-fp),f);
						length -= (&filebuf[filebuf_size]-fp);

						fwrite(filebuf,1,filebuf_size,f_out);
						fp = filebuf;
					}
					if(length > 0)
					{
						fread(fp,1,length,f);
						fp += length;
					}

					art_file_header[fileno].n_articles++;
				}

				prev_displ = displ;
				displ += (head_length + sizeof(art_head));
				fseek(f,displ,SEEK_SET);
			}

			if(fp>filebuf)
			{
				fwrite(filebuf,1,fp-filebuf,f_out);
			}
			fp = filebuf;
		}

		if(errors)
		{
			werr(0,"Failed to process file 'articles%d'",fileno);
		}

		displ = (int)ftell(f_out);
		fclose(f_out);
		fclose(f);

		remove(fname2);    /* articles98 */

		if(errors == 0)
		{
			art_file_header[fileno].free_space = 0;
			art_file_header[fileno].size = displ;

			/* delete original file and rename new one */

			sprintf(fname,"%s.articles%d",pluto_articles,fileno);
			sprintf(fname2,"%s.articles99",pluto_articles);

			if(get_filelength(fname2) > 0)
			{
				/* make sure articles99 exists */
				remove(fname);
				rename(fname2,fname);
			}
		}
	}

	flex_free((flex_ptr)&ixlist_article);
	if(text_temp->text_base != NULL)
		flex_free((flex_ptr)&text_temp->text_base);

	free(filebuf);
	free(text_temp);
	visdelay2_end();
	clear_lock_lists(1);

	/* now rebuild the index */
	close_all_list_windows(0);
	article_file_recover(0,0);

	cardfile_free(0);
	cardfile_load(0);

	return(errors);
}   /* end of article_file_set_references */





#define N_COMPACT_BUF 0x4000  /* 16k */




int article_file_compact(int cf, int fileno, int min_free_space)
/**************************************************************/
/* Go through an article file and recover waste space.  Compact the
   data and update the pointers in the card file.
   Need to make article_ixlist first */
{
	FILE *f;
	FILE *f_out;
	int  displ;
	int new_displ;
	int seek_displ;
	int prev_displ;
	int error_displ;
	int key;
	int key2;
	int  ix;
	int  length;
	int  head_length;
	CARD *cptr;
	CARD *prev_cptr=NULL;
	CARD *cptr_trace = NULL;
	int  errors=0;
	int  box;
	ART_HEAD art_head;
	ART_FILE_HEADER file_header;
	CARD card;
	char fname[256];
	char buf[N_COMPACT_BUF];

	int filebuf_size;
	char *filebuf=NULL;
	char *fp;


	if((art_file_header[fileno].version == 0) ||
			(art_file_header[fileno].free_space < min_free_space))
		return(0);

	if((f = article_fopen(cf,fileno,"r")) == NULL) return(0);
	sprintf(fname,"%s.articles99",cardfile[cf].path);
	if((f_out = fopen_werr(fname,"w",NULL)) == NULL)
	{
		fclose(f);
		return(0);
	}

	cardfile_changed[cf] = 1;

	filebuf_size = 0x20000;
	filebuf = malloc(filebuf_size);
	fp = filebuf;   /* may be NULL if malloc fails */

	art_file_header[fileno].n_articles = 0;

	memset(&file_header,0,sizeof(file_header));
	file_header.version = ART_FILE_VERSION;
	fread(&file_header,sizeof(file_header),1,f);
	fwrite(&file_header,sizeof(file_header),1,f_out);
	displ = (int)ftell(f);

	/* ixlist_article is sorted by article address */
	key = (fileno << 24) + displ;
	key2 = (fileno + 1) << 24;

	/* find the start of ixlist entries for the specified article file */
	for(ix=0; ix < list_fr[0].n_entries; ix++)
	{
		cptr = (CARD *)((ixlist_article[ix] & IXLIST_MASK) << 2);
		if(cptr->addr >= key)
			break;
	}

	if((ix < list_fr[0].n_entries) || (cptr->addr < key2))
	{
		while(!feof(f))
		{

			if((ix & 7) == 0)
				call_event_process(0);

			displ = (int)ftell(f);
			if(fread(&art_head,sizeof(art_head),1,f)==0) break;
			if(feof(f))
				break;

			head_length = art_head.length & ART_LENGTH_MASK;
			if(art_head.magic != MAGIC)
			{
				error_displ = displ;
				seek_displ = displ & ~4;

				for(;;)
				{
					seek_displ += 4;
					fseek(f,seek_displ,SEEK_SET);
					if(fread(&art_head,sizeof(art_head),1,f)==0) break;
					if(art_head.magic == MAGIC)
					{
						break;
					}
				}

				if(f_errors != NULL)
					fprintf(f_errors,"Article file %d/%2d is corrupt at %6x continue at %6x\n",cf,fileno,error_displ,seek_displ);
				error_log_count++;
				errors=2;
				break;
			}

			prev_cptr = cptr;

			key = (fileno << 24) + displ;
			while((ix < list_fr[0].n_entries) && (cptr->addr < key))
			{
				cptr_trace = cptr;  /* just for testing */
				cptr = (CARD *)((ixlist_article[++ix] & IXLIST_MASK) << 2);
			}

			if((ix >= list_fr[0].n_entries) || (cptr->addr > key))
			{
				if(art_head.length >= 0)
				{
					fread(&card,1,16,f);
					box = card.date_box >> 26;
					if(box < BOX_BIN)
					{
						if(f_errors != NULL)
							fprintf(f_errors,"Card file at %p and Article file %2d don't match at %6x\n",
									((char *)cptr),fileno,displ);
						error_log_count++;
						errors=1;
						dont_save_cardfile = 1;
						break;
					}
				}
			}
			else if(art_head.length < 0)
			{
				/* mark this record as deleted, it will be removed from the
				   cardfile when it is saved when the program terminates */

				cptr->date_box = (cptr->date_box & ~BOX_MASK) + ((unsigned)BOX_DELETED << 26);
			}
			else
			{
				/* this block is not garbage, copy to new file */

				new_displ = (int)ftell(f_out);
				if(fp != NULL)
				{
					new_displ += (fp - filebuf);
				}

				fread(&card,1,16,f);
				length = card.alength & ART_LENGTH_MASK;
				length = (length + 3) & ~3;   /* round up to word boundary */

				if(length > (art_head.length & ART_LENGTH_MASK))
				{
					if(f_errors != NULL)
						fprintf(f_errors,"Wrong article length value in file %2d at %6x",fileno,new_displ);
					error_log_count++;
					errors=3;
					break;
				}
				art_head.length = length;

				/* update the card file */
				cptr->addr = (cptr->addr & ~ART_ADDR_MASK) + new_displ;
				card.addr = cptr->addr;
				length -= 16;

				if(fp != NULL)
				{
					if((&filebuf[filebuf_size]-fp) < (sizeof(art_head)+16))
					{
						fwrite(filebuf,1,fp-filebuf,f_out);
						fp = filebuf;
					}
					memcpy(fp,&art_head,sizeof(art_head));
					fp += sizeof(art_head);
					memcpy(fp,&card,16);
					fp += 16;
				}
				else
				{
					fwrite(&art_head,1,sizeof(art_head),f_out);
					fwrite(&card,1,16,f_out);
				}

				if(fp != NULL)
				{
					while(length >= (&filebuf[filebuf_size]-fp))
					{
						fread(fp,1,(&filebuf[filebuf_size]-fp),f);
						length -= (&filebuf[filebuf_size]-fp);

						fwrite(filebuf,1,filebuf_size,f_out);
						fp = filebuf;
					}
					if(length > 0)
					{
						fread(fp,1,length,f);
						fp += length;
					}
				}
				else
				{
					while(length >= N_COMPACT_BUF)
					{
						fread(buf,1,N_COMPACT_BUF,f);
						fwrite(buf,1,N_COMPACT_BUF,f_out);
						length -= N_COMPACT_BUF;
					}
					if(length > 0)
					{

						fread(buf,1,length,f);
						fwrite(buf,1,length,f_out);
					}
				}

				art_file_header[fileno].n_articles++;
			}

			prev_displ = displ;
			displ += (head_length + sizeof(art_head));
			fseek(f,displ,SEEK_SET);
		}

		if(fp>filebuf)
		{
			fwrite(filebuf,1,fp-filebuf,f_out);
		}
	}

	displ = (int)ftell(f_out);
	fclose(f);
	fclose(f_out);

	if(filebuf != NULL)
	{
		free(filebuf);
	}


	if(errors == 0)
	{
		art_file_header[fileno].free_space = 0;
		art_file_header[fileno].size = displ;

		/* delete original file and rename new one */

		sprintf(fname,"%s.articles%d",cardfile[cf].path,fileno);
		sprintf(buf,"%s.articles99",cardfile[cf].path);

		if(get_filelength(buf) > 0)
		{
			/* make sure articles99 exists */
			remove(fname);
			rename(buf,fname);
		}
	}

	return(errors);
}   /* end of article_file_compact */




void article_file_compact_all()
/*****************************/
{
	int  fileno;
	int  total;
	int  acc=0;

	if(query("Compact articles files.  This may take a long time. Continue?",NULL)==0)
		return;

	dont_save_cardfile = 0;
	error_log_open();

	visdelay2_begin();

	box_close_all_external();

	if(article_make_index(&list_fr[0],&ixlist_article) != NULL)
	{
		total = article_file_total_size(0) / 256;

		for(fileno=0; fileno<N_ART_FILES; fileno++)
		{
			visdelay2_percent(acc*100/total);
			acc += (article_file_size(0,fileno) / 256);

			article_file_compact(0,fileno,MIN_ART_COMPACT);
		}

		flex_free((flex_ptr)&ixlist_article);
	}
	set_boxlist_extent(7);
	visdelay2_end();
	announce_message(msgs_lookup("Y11"),1);   // article files compacted

	write_log_message("Compact all");

	error_log_report("errors found whie compacting articles");

}   /* end of article_file_compact_all */




int card_size_old2(CARD_OLD *cptr)
/********************************/
/* Find size of card data, as written into the article file */
{
	int  length;
	int  d_end;
	static int d_end_tab[] = {0,0x120,0x140,0x180};

	if(cptr == NULL)
		return(0);

	d_end = cptr->d_end;
	if(d_end <= 3)
	{
		d_end = d_end_tab[d_end];
	}
	length = cptr->data - (char *)cptr + d_end;
	length = (length + 3) & ~0x3;   /* align to word boundary */
	return(length);
}   /* end of card_size_old */




int card_size_old3(CARD_OLD3 *cptr)
/*********************************/
/* Find size of card data, as written into the article file */
{
	int  length;
	int  d_end;
	static int d_end_tab[] = {0,0x120,0x140,0x180};

	if(cptr == NULL)
		return(0);

	d_end = cptr->d_end;
	if(d_end <= 3)
	{
		d_end = d_end_tab[d_end];
	}
	length = cptr->data - (char *)cptr + d_end;
	length = (length + 3) & ~0x3;   /* align to word boundary */
	return(length);
}   /* end of card_size_old3 */





int article_file_repair(int cf, char *fname, int length_acc,
						int length_total, int fileno)
/*******************************************************************/
{
	FILE *f;
	FILE *f_out;
	long int  displ;
	long int  next_displ;
	int  new_displ;
	int  length;
	int  data_length;
	long int  file_length;
	int  unsquash_length;
	int  card_fileno;
	int  error_count = 0;
	long int  x;
	int  i;
	static int  size;
	static int  old_size;
	int cd_size;
	ART_HEAD art_head;
	ART_HEAD art_head2;
	ART_FILE_HEADER file_header;

	static CARD card;
	static CARD_OLD card_old;

	static CARD_OLD3 card_old3;

	static char buf[N_COMPACT_BUF];


	/* upgrade v1 to v2, change meaning of status field */
	int upgrade_2=0;
	int upgrade_3=0;


	if((f = fopen(fname,"r")) == NULL) return(0);

	sprintf(buf,"%s.articles99",cardfile[cf].path);
	if((f_out = fopen_werr(buf,"w",NULL)) == NULL)
	{
		fclose(f);
		return(0);
	}

	file_length = get_filelength(fname);

	memset(&file_header,0,sizeof(file_header));
	file_header.version = ART_FILE_VERSION;
	fread(&file_header,sizeof(file_header),1,f);

	upgrade_2 = 0;
	if(file_header.version < 3)
	{
		upgrade_2 = 1;
		file_header.version = 3;
	}

	if(file_header.version < 4)
	{
		upgrade_3 = 1;
		file_header.version = ART_FILE_VERSION;
	}

	fwrite(&file_header,sizeof(file_header),1,f_out);

	while(!feof(f))
	{
		call_event_process(0);

		displ = ftell(f);
		if(fread(&art_head,sizeof(art_head),1,f)==0) break;
		if(feof(f))
			break;


		if(art_head.magic == MAGIC)
		{
			/* is the length correct ?
			   Look to see whether there is a valid article header (or eof) after
			   this specified length */

			length = (art_head.length & ART_LENGTH_MASK) + sizeof(ART_HEAD);
			next_displ = displ + length;

			art_head2.magic = 0;
			if(next_displ == file_length)
			{
				art_head2.magic = MAGIC;   /* this was the last article in the file */
			}
			else if(next_displ < file_length)
			{
				fseek(f,next_displ,SEEK_SET);
				fread(&art_head2,sizeof(art_head),1,f);
			}

			if(art_head2.magic == MAGIC)
			{
				/* first article header is correct, write out article, unless it is binned */
				if(art_head.length > 0)
				{
					fseek(f,displ+sizeof(ART_HEAD),SEEK_SET);

					size = 22;
					fread(&card,1,22,f);

					length = card.alength & ART_LENGTH_MASK;
					length = (length + 3) & ~3;   /* round up to word boundary */
					data_length = length - 22;

					/*
					 pack and unpack
					 STATUS_BIT_SQUASHED
					 all time routines
					 expiry date
					*/

					if(upgrade_2)
					{
						memcpy(&card_old,&card,22);
						card.d_end = card_old.d_end;
						size = card_size(&card);   /* new size */
						old_size = card_size_old2(&card_old);


						fseek(f,displ+sizeof(ART_HEAD),SEEK_SET);
						fread(&card_old,1,old_size,f);

						card.date_box = card_old.date_box & 0xfc000000;  /* box */
						card.date_box |= ((card_old.date_box & 0xffffff) << 2 );  /* date */
						card.n_refs = 0;
						card.status = card_old.status;
						i = (card_old.date_box & 0x03000000) >> 18;
						card.user = i;

						memcpy(&card.n_cats,&card_old.n_cats,size-16);
						card.alength += (size - old_size);

						length = card.alength & ART_LENGTH_MASK;
						length = (length + 3) & ~3;   /* round up to word boundary */
						data_length = length - size;

						art_head.length += 4;  /* avoid test below from failing */
					}

					if(upgrade_3)
					{
						memcpy(&card_old3,&card,22);
						card.d_end = card_old3.d_end;
						size = card_size(&card);   /* new size */
						old_size = card_size_old3(&card_old3);

						fseek(f,displ+sizeof(ART_HEAD),SEEK_SET);
						fread(&card_old3,1,old_size,f);

						memcpy(card.data,card_old3.data,size-22);
						card.n_refs = 0;
						card.score = 0;
						card.msgid = 0;
						card.parent = 0;

						card.alength += (size - old_size);

						length = card.alength & ART_LENGTH_MASK;
						length = (length + 3) & ~3;   /* round up to word boundary */
						data_length = length - size;

						art_head.length += 12;  /* avoid test below from failing */
					}



					if((card.status & STATUS_MASK2) == 0x7)
					{
						/* change Hidden to Read - WHY???? */
						card.status = card.status & ~STATUS_MASK | STATUS_READ;
					}

					unsquash_length = 0x7fffffff;
					if((!upgrade_2) && (!upgrade_3) && (card.user & SQUASHED_MASK))
					{
						x = (int)ftell(f);
						cd_size = card_size(&card);
						fseek(f,displ+sizeof(ART_HEAD)+cd_size,SEEK_SET);
						fread(&unsquash_length,1,4,f);
						fseek(f,x,SEEK_SET);
					}

					card_fileno = card.addr >> 24;
					if(cf > 0)
						card_fileno &= 0x1f;

					if(unsquash_length < (length-cd_size-4))
					{
						x = displ+sizeof(ART_HEAD)+size;
						error_count++;
					}
					else if((length <= (art_head.length & ART_LENGTH_MASK)) && (card_fileno == fileno))
					{
						if((card.date_box >> 26) < BOX_BIN)
						{
							visdelay2_percent((int)((length_acc+displ/256)*100/length_total));

							art_head.length = length;

							new_displ = (int)ftell(f_out);
							fwrite(&art_head,1,sizeof(art_head),f_out);

							/* update the card file */
							card.addr = card.addr & ~ART_ADDR_MASK + new_displ;
							fwrite(&card,1,size,f_out);

							while(data_length >= N_COMPACT_BUF)
							{
								fread(buf,1,N_COMPACT_BUF,f);
								fwrite(buf,1,N_COMPACT_BUF,f_out);
								data_length -= N_COMPACT_BUF;
							}
							if(data_length > 0)
							{

								fread(buf,1,data_length,f);
								fwrite(buf,1,data_length,f_out);
							}
						}
					}
					else
					{
						error_count++;
					}
				}

				if(next_displ >= file_length)
					break;

				fseek(f,next_displ,SEEK_SET);
			}
			else
			{
				error_count++;
				fseek(f,displ+1,SEEK_SET);   /* continue searching */
			}
		}
		else
		{
			fseek(f,displ+1,SEEK_SET);
		}
	}
	fclose(f);
	fclose(f_out);

	/* delete original file and rename new one */

	sprintf(buf,"%s.articles99",cardfile[cf].path);

	os_swi4(0x29, 27, (int)fname, 0, 0);            /* OS_FSControl  Wipe */
	os_swi3(0x29, 25, (int)buf, (int)fname);        /* OS_FSControl  Rename */

	return(error_count);
}   /* end of article_file_repair */




int article_files_repair(int cf, int all_files)
/*********************************************/
/* Compact all article files */
{
	int  fileno;
	int  length_total=0;
	int  length_acc=0;
	int  length;
	int  error_count = 0;
	FILE *f;
	dbox d;
	int fileno_start=0;
	int fileno_end;
	ART_FILE_HEADER file_header;
	char buf[256];
	static char *msg = "Repair/Compact Articles.  This may take a long time.  Continue?";

	if(cf==0)
		strcpy(buf,msg);
	else
		sprintf(buf,"%s: %s",box_table[cf_to_box(cf)].name,msg);

	if(query(buf,NULL)==0)
		return(1);

	if(cf==0)
		fileno_end = N_ART_FILES-1;
	else
		fileno_end = N_ART_FILES_X-1;

	if(all_files == 0)
	{
		d = dbox_new("ArticleFile");
		dbox_setnumeric(d,3,fileno_start);
		dbox_setnumeric(d,4,fileno_end);
		dbox_show(d);
		if(dbox_fillin(d) == 0)
		{
			fileno_start = dbox_getnumeric(d,3);
			fileno = dbox_getnumeric(d,4);
			if(fileno < fileno_end)
				fileno_end = fileno;
			dbox_dispose(&d);
		}
		else
		{
			dbox_dispose(&d);
			return(2);
		}
	}

	visdelay2_begin();
	set_lock_lists(1,NULL);

	for(fileno=fileno_start; fileno <= fileno_end; fileno++)
	{
		sprintf(buf,"%s.articles%d",cardfile[cf].path,fileno);
		length = get_filelength(buf);
		if(length <= 0)
		{
			memset(&file_header,0,sizeof(file_header));
			file_header.version = ART_FILE_VERSION;
			f = fopen(buf,"w");
			fwrite(&file_header,1,sizeof(file_header),f);
			fclose(f);
		}
		length_total += length;
	}

	for(fileno=fileno_start; fileno <= fileno_end; fileno++)
	{
		sprintf(buf,"%s.articles%d",cardfile[cf].path,fileno);
		length = get_filelength(buf);
		error_count += article_file_repair(cf,buf,length_acc/256,length_total/256,fileno);

		length_acc += length;
	}
	clear_lock_lists(1);
	visdelay2_end();


	if((text_view != NULL) && (text_view->cptr != NULL))
	{
		/* close the article viewer window */
		text_window_close(text_view);
		close_article_window(text_view);
	}


	close_all_list_windows(cf);
	article_file_recover(cf,0);

	cardfile_free(cf);
	cardfile_load(cf);

	sprintf(buf,msgs_lookup("Y13"),error_count);   // repair completed
	announce_message(buf,1);
	write_log_message(buf);
	werr(0,buf);
	set_boxlist_extent(7);
	return(0);
}   /* end of article_files_repair */




int article_files_compact_most(int control, int minimum, int *error)
/******************************************************************/
/* Compact the article files with the most free space */
/* Control bit 0:  don't compact files which have just read news/mail
               3:  create ixlist_article and free it afterwards */
{
	int  fileno;
	int  max=0;
	int  max_fileno = -1;
	int  garbage;

	dont_save_cardfile = 0;
	garbage = 0;

	for(fileno=0; fileno<N_ART_FILES; fileno++)
	{
		garbage += art_file_header[fileno].free_space;

		if(compact_attempted[fileno])
			continue;   /* don't attempt the same file twice */

		if((control & 1) && (appended_art_file[fileno]!=0))
			continue;   /* articles have been debatched into this file previously */

		if(art_file_header[fileno].free_space > max)
		{
			max =art_file_header[fileno].free_space;
			max_fileno = fileno;
		}
	}

	if(garbage < minimum)
		return(-1);

	if(max_fileno == -1)
	{
		/* all article files are marked as receiving news/mail */
	}
	else
	{
		compact_attempted[max_fileno] = 1;

		if(max > MIN_ART_COMPACT)
		{
			if(control & 8)
			{
				/* external boxes should have been closed by this point */

				/* sort index into article number order */
				if(article_make_index(&list_fr[0],&ixlist_article)==NULL)
					return(-1);
			}

			set_lock_lists(1,NULL);
			*error = article_file_compact(0,max_fileno,MIN_ART_COMPACT);
			clear_lock_lists(1);

			if(control & 8)
				flex_free((flex_ptr)&ixlist_article);

			return(max_fileno);
		}
	}
	return(-1);
}   /* article_files_compact_most */




void log_article(FILE *f, CARD *cptr)
/***********************************/
{
	char *p;
	char buf[40];

	strcpy_printable(buf,&cptr->data[cptr->d_author],sizeof(buf));
	p = &buf[1];
	while((*p != '<') && (*p != 0))
		p++;
	*p = 0;

	fprintf(f," %s  %s  %s: %s\n",decode_date(cptr->date_box,0),expand_source(cptr->source,3),buf,&cptr->data[cptr->d_title]);
}   /* end of log_article */



void article_delete_duplicates()
/******************************/
/* Delete articles with duplicate message-ids */
{
	int  ix;
	FOLDREC *fr;
	CARD *cptr1;
	CARD *cptr2;
	int  size1;
	int  size2;
	int  size_offset;
	int  diff;
	int  n_cards=0;
	int  expire_box = BOX_BIN;
	int  result;
	int  update;
	FILE *f_log=NULL;
	char buf[48];
	int maxdiff;
	int logging=1;


	static status_priority[8] = {4,5,6,2,3,0,1,7};

	if(check_lock_lists(0xffff) != 0)
		return;

	visdelay2_begin();
	set_lock_lists(3,NULL);

	if(logging)
	{
		write_log_message("\n*** Delete duplicates");
		f_log = fopen_pluto("Backup.log","a");
	}

	fr = &list_fr[0];
	sort_for_duplicates(fr);   /* Sort on Length, Date, Title */

	cptr2 = (CARD *)((fr->ixlist[0] & IXLIST_MASK) << 2);
	size2 = card_size(cptr2);

	size_offset = 18 + 3;  /* &cptr1->d_title - &cptr1->addr, remove possible padding from the end */

	for(ix=1; ix<fr->n_entries; ix++)
	{
		cptr1 = cptr2;
		size1 = size2;

		cptr2 = (CARD *)((fr->ixlist[ix] & IXLIST_MASK) << 2);
		size2 = card_size(cptr2);

		/* must be in the same Box and have same date/time */
		if((cptr1->date_box == cptr2->date_box) && (cptr1->msgid == cptr2->msgid) && ((cptr1->date_box >> 26) < BOX_BIN))
		{
			diff = cptr1->alength - cptr2->alength;

			if(cptr1->msgid == 0)
				maxdiff = 2;  /* no msgid, use stricter article length match */
			else
				maxdiff = 100;

			if((size1 == size2) && (diff >= -maxdiff) && (diff <= maxdiff))
			{
				call_event_process(0);

				if(memcmp(&cptr1->d_title,&cptr2->d_title,size1-size_offset)==0)
				{
					n_cards++;

					update = 0;
					if(status_priority[cptr1->status & STATUS_MASK] <
							status_priority[cptr2->status & STATUS_MASK])
					{
						/* keep locked and marked status */
						cptr2->status = (cptr2->status & ~STATUS_MASK) | (cptr1->status & STATUS_MASK);
						update = 1;
					}
					if((cptr1->status & STATUS_BIT_REPLIED) && !(cptr2->status & STATUS_BIT_REPLIED))
					{
						/* keep Replied status */
						cptr2->status |= STATUS_BIT_REPLIED;
						update = 1;
					}

					if(update)
					{
						article_update(NULL,cptr2,NULL,0,0);
					}


					if(f_log != NULL)
					{
						log_article(f_log,cptr1);
					}

					result = 0;
					if(expire_box != BOX_BIN)
					{
						cptr1->date_box = (cptr1->date_box & ~BOX_MASK) + (expire_box << 26);

						if(box_table[expire_box].flags & BOX_STRIP_HEADERS)
							result = article_delete_internet_header(NULL,cptr1,1);
						else
							result = article_update(NULL,cptr1,NULL,0,0);
					}
					else
					{
						result = article_delete(NULL,cptr1,1,6);   /* and box -> BIN */
					}

					if(result != 0)
					{
						werr(0,"Abandoned");
						break;
					}
				}
			}
		}
	}

	clear_lock_lists(3);
	visdelay2_end();
	set_boxlist_extent(3);

	sprintf(buf,msgs_lookup("Y14"),n_cards);    // deleted n duplicate articles
	announce_message(buf,1);
	user_message(buf);

	if(f_log != NULL)
	{
		fprintf(f_log,"%s\n\n",buf);
		fclose(f_log);
	}
}   /* end of article_delete_duplicates */




void article_file_ressurrect(int cf)
/**********************************/
/* Unhide "hidden" articles in the Bin */
{
	int  ix;
	int  n_entries;
	CARD *cptr;
	int  box;
	unsigned int *ixlist;

	ixlist = list_fr[0].ixlist;
	n_entries = list_fr[0].n_entries;

	visdelay2_begin();

	for(ix=0; ix<n_entries; ix++)
	{
		cptr = (CARD *)((ixlist[ix] & IXLIST_MASK) << 2);

		box = cptr->date_box >> 26;
		if((box == BOX_DELETED) || (box_to_cf(box) != cf))
			continue;

		if((cptr->status & STATUS_MASK2) == STATUS_HIDDEN)
		{
			/* change Hidden status back to Read */
			cptr->status = (cptr->status & ~STATUS_MASK2) | STATUS_READ;

			article_update(NULL,cptr,NULL,0,0);
			call_event_process(0);
		}
	}
	visdelay2_end();
	set_boxlist_extent(3);   /* recalc totals */
}   /* end of article_file_ressurrect */




int artfiles_count_garbage()
/***********************/
{
	int  i;
	int  garbage=0;

	for(i=0; i<N_ART_FILES; i++)
		garbage += art_file_header[i].free_space;

	return(garbage);
}   /* end of article_garbage */




void artfile_statistics(int cf)
/**************************/
{
	int  i;
	int  n_entries;
	unsigned int *ixlist;
	CARD *cptr;
	int  articles=0;
	int  used=0;
	int  wasted=0;
	int  n_art_files;
	int  start;
	FOLDREC *fr;
	dbox d;
	double x;
	char buf[24];

	if(cf==0)
		n_art_files = N_ART_FILES;
	else
		n_art_files = N_ART_FILES_X;

	d = dbox_new("Stats");

	if(cf == -1)
	{
		/* count articles in the selected boxes */
		fr = &list_fr[0];
		n_entries = fr->n_entries;
		ixlist = fr->ixlist;

		for(i=0; i<n_entries; i++)
		{
			cptr = (CARD *)((ixlist[i] & IXLIST_MASK) << 2);
			if(box_selected[cptr->date_box >> 26])
			{
				articles++;
				used += cptr->alength & ART_LENGTH_MASK;
			}
		}
		dbox_fadefield(d,2);
		dbox_fadefield(d,5);
	}
	else
	{
		start = cf_filenum(cf,0);
		for(i=start; i<(start+n_art_files); i++)
		{
			articles += art_file_header[i].n_articles;
			used += art_file_header[i].size;
			wasted += art_file_header[i].free_space;
		}

		x = (double)wasted;
		sprintf(buf,"%.2f",x/1048576);
		dbox_setfield(d,2,buf);
	}

	dbox_setnumeric(d,0,articles);

	x = (double)(used-wasted);
	sprintf(buf,"%.2f",x/1048576);
	dbox_setfield(d,1,buf);

	dbox_show(d);
	dbox_fillin(d);
	dbox_dispose(&d);

}   /* end of artfile_statistics */



void article_exit()
/*****************/
/* Compacts up to 4 articles files */
{
	int  count;
	int  i;
	unsigned int *ixlist;
	int  n_entries;
	CARD *cptr;
	int  box;
	unsigned int *new_ixlist;
	int  control;
	double garbage1;
	double garbage2;
	int  minimum;
	int  error;
	int  any_errors=0;
	int  n_compact;
	int  new_size;
	static char *none = " none";
	FILE *f;
	FOLDREC *fr;
	int  cf;
	int any_external=0;
	char buf[80];

	if(cardfile[0].base == NULL)
	{
		return;   /* cardfile has not been loaded */
	}

	visdelay2_begin();
	fr = &list_fr[0];

	for(cf=1; cf < (N_BOXES_EXTERN+1); cf++)
	{
		if((cardfile[cf].path[0] != 0) && (cardfile[cf].base!=NULL))
		{
			if(cardfile_changed[cf])
			{
				any_external += box_save_external(cf_to_box(cf),1);
			}
			else
			{
				for(i=0; i<N_ART_FILES_X; i++)
				{
					any_external += art_file_header[cf_filenum(cf,i)].n_articles;
				}
			}

			/* can't free cardfile[cf].base yet because we need to check
			   box of all articles below */
		}
	}

	/* delete external box articles from the main index */
	if(any_external)
	{
		ixlist = fr->ixlist;
		n_entries = fr->n_entries;

		/* how many articles left after those saved in external boxes ? */
		new_size = n_entries - any_external + 4;

		if((new_ixlist = malloc(new_size * sizeof(int))) != NULL)
		{
			count = 0;
			for(i=0; i<n_entries; i++)
			{
				cptr = (CARD *)((ixlist[i] & IXLIST_MASK) << 2);
				if(((box = cptr->date_box >> 26) < BOX_EXTERN) || (box == BOX_BIN))
				{
					if((cptr->status & STATUS_MASK2) != STATUS_HIDDEN)
					{
						new_ixlist[count++] = ixlist[i];

						if(count >= new_size)
						{
							/* calculated the required ixlist size wrongly */
							werr(0,"Article exit error 1");
							return;
						}
					}
				}
			}
		}
		fr->ixlist = new_ixlist;
		fr->n_entries = count;

		/* now free the cptr data for external boxes */
		for(cf=1; cf < (N_BOXES_EXTERN+1); cf++)
		{
			if((cardfile[cf].path[0] != 0) && (cardfile[cf].base!=NULL))
			{
				free(cardfile[cf].base);
			}
		}
	}


	/* sort index into article number order, needed by article_file_compact */
	sort_card_file = 0;
	qsortG(fr->ixlist,fr->n_entries,sizeof(int *),article_sorter);
	ixlist_article = fr->ixlist;

	memset(compact_attempted,0,sizeof(compact_attempted));

	garbage1 = (double)artfiles_count_garbage();
	get_today_date2(buf);
	f = fopen_pluto("Backup.log","a");
	if(f != NULL)
		fprintf(f,"%s  Exit, compact",buf);

	if(cardfile_changed[0])
		minimum = 300000;
	else
		minimum = 500000;

	control = 1;
	n_compact = 4;
	if(garbage1 > 10000000)
		n_compact++;

	error_log_open();

	for(count=0; count<n_compact; count++)
	{
		i = article_files_compact_most(control,minimum,&error);   /* until garbage drops below the lower limit */
		if(error == 1)
			any_errors = 1;   /* index didn't match articles files */

		if(i >= 0)
		{
			if(f != NULL)
			{
				fprintf(f," #%d",i);
				if(error)
					fprintf(f," (errors)");
			}
			none = "";
		}
		minimum = 1500000;  /* only compact more than 1 file if garbage is more than this */
		control = 0;
	}
	garbage2 = (double)artfiles_count_garbage();
	if(f != NULL)
	{
		fprintf(f,"%s  %.2f->%.2f MB\n",none,garbage1/1048576,garbage2/1048576);
		fclose(f);
	}

	if(cardfile_changed[0] && (dont_save_cardfile==0))
	{
		cardfile_save(0,0);
	}
	else
	{
		cardfile_save(0,4);  // just timestamp it (to avoif rebuilt later)
	}

	visdelay2_end();

	error_log_report("errors found while compacting articles");

}  /* end of article_exit */


