# HG changeset patch

# User Timo Sirainen <tss@iki.fi>

# Date 1416386277 -32400

# Node ID 80ed82a93c1af5f6885e75a34007eb52d0692a8d

# Parent  31262a892ba7302bfea6e70e17d3acd468736d70

mbox: Added more (and earlier) detection for errors in mbox istreams.

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/istream-raw-mbox.c

--- a/src/lib-storage/index/mbox/istream-raw-mbox.c	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/istream-raw-mbox.c	Wed Nov 19 17:37:57 2014 +0900

@@ -506,7 +506,8 @@

 	return rstream->from_offset;

 }

-uoff_t istream_raw_mbox_get_header_offset(struct istream *stream)

+int istream_raw_mbox_get_header_offset(struct istream *stream,

+				       uoff_t *hdr_offset_r)

 {

 	struct raw_mbox_istream *rstream =

 		(struct raw_mbox_istream *)stream->real_stream;

@@ -520,13 +521,17 @@

 		i_error("Unexpectedly lost From-line from mbox file %s at "

 			"%"PRIuUOFF_T, i_stream_get_name(stream),

 			rstream->from_offset);

-		return (uoff_t)-1;

+		return -1;

 	}

+	if (stream->stream_errno != 0)

+		return -1;

-	return rstream->hdr_offset;

+	*hdr_offset_r = rstream->hdr_offset;

+	return 0;

 }

-uoff_t istream_raw_mbox_get_body_offset(struct istream *stream)

+int istream_raw_mbox_get_body_offset(struct istream *stream,

+				     uoff_t *body_offset_r)

 {

 	struct raw_mbox_istream *rstream =

 		(struct raw_mbox_istream *)stream->real_stream;

@@ -534,8 +539,10 @@

 	i_assert(rstream->seeked);

-	if (rstream->body_offset != (uoff_t)-1)

-		return rstream->body_offset;

+	if (rstream->body_offset != (uoff_t)-1) {

+		*body_offset_r = rstream->body_offset;

+		return 0;

+	}

 	offset = stream->v_offset;

 	i_stream_seek(stream, rstream->hdr_offset);

@@ -551,27 +558,30 @@

 			} else {

 				i_assert(rstream->body_offset != (uoff_t)-1);

 			}

-			break;

+			return -1;

 		}

 	}

 	i_stream_seek(stream, offset);

-	return rstream->body_offset;

+	*body_offset_r = rstream->body_offset;

+	return 0;

 }

-uoff_t istream_raw_mbox_get_body_size(struct istream *stream,

-				      uoff_t expected_body_size)

+int istream_raw_mbox_get_body_size(struct istream *stream,

+				   uoff_t expected_body_size,

+				   uoff_t *body_size_r)

 {

 	struct raw_mbox_istream *rstream =

 		(struct raw_mbox_istream *)stream->real_stream;

 	const unsigned char *data;

 	size_t size;

-	uoff_t old_offset, body_size, next_body_offset;

+	uoff_t old_offset, body_offset, body_size, next_body_offset;

 	i_assert(rstream->seeked);

 	i_assert(rstream->hdr_offset != (uoff_t)-1);

-	(void)istream_raw_mbox_get_body_offset(stream);

+	if (istream_raw_mbox_get_body_offset(stream, &body_offset) < 0)

+		return -1;

 	body_size = rstream->mail_size == (uoff_t)-1 ? (uoff_t)-1 :

 		rstream->mail_size - (rstream->body_offset -

 				      rstream->hdr_offset);

@@ -580,8 +590,10 @@

 		/* if we already have the existing body size, use it as long as

 		   it's >= expected body_size. otherwise the previous parsing

 		   may have stopped at a From_-line that belongs to the body. */

-		if (body_size != (uoff_t)-1 && body_size >= expected_body_size)

-			return body_size;

+		if (body_size != (uoff_t)-1 && body_size >= expected_body_size) {

+			*body_size_r = body_size;

+			return 0;

+		}

 		next_body_offset = rstream->body_offset + expected_body_size;

 		/* If header_missing_eoh is set, the message body begins with

@@ -600,21 +612,27 @@

 			rstream->mail_size =

 				next_body_offset - rstream->hdr_offset;

 			i_stream_seek(stream, old_offset);

-			return expected_body_size;

+			*body_size_r = expected_body_size;

+			return 0;

 		}

 		/* invalid expected_body_size */

 	}

-	if (body_size != (uoff_t)-1)

-		return body_size;

+	if (body_size != (uoff_t)-1) {

+		*body_size_r = body_size;

+		return 0;

+	}

 	/* have to read through the message body */

 	while (i_stream_read_data(stream, &data, &size, 0) > 0)

 		i_stream_skip(stream, size);

 	i_stream_seek(stream, old_offset);

+	if (stream->stream_errno != 0)

+		return -1;

 	i_assert(rstream->mail_size != (uoff_t)-1);

-	return rstream->mail_size -

+	*body_size_r = rstream->mail_size -

 		(rstream->body_offset - rstream->hdr_offset);

+	return 0;

 }

 time_t istream_raw_mbox_get_received_time(struct istream *stream)

@@ -651,13 +669,15 @@

 	return rstream->crlf_ending;

 }

-void istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size)

+int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size)

 {

 	struct raw_mbox_istream *rstream =

 		(struct raw_mbox_istream *)stream->real_stream;

 	uoff_t body_size;

-	body_size = istream_raw_mbox_get_body_size(stream, expected_body_size);

+	if (istream_raw_mbox_get_body_size(stream, expected_body_size,

+					   &body_size) < 0)

+		return -1;

 	rstream->mail_size = (uoff_t)-1;

 	rstream->received_time = rstream->next_received_time;

@@ -678,6 +698,7 @@

 	rstream->eof = FALSE;

 	rstream->istream.istream.eof = FALSE;

+	return 0;

 }

 int istream_raw_mbox_seek(struct istream *stream, uoff_t offset)

@@ -698,10 +719,8 @@

 	/* if seeked is FALSE, we unlocked in the middle. don't try to use

 	   any cached state then. */

 	if (rstream->mail_size != (uoff_t)-1 && rstream->seeked &&

-	    rstream->hdr_offset + rstream->mail_size == offset) {

-		istream_raw_mbox_next(stream, (uoff_t)-1);

-		return 0;

-	}

+	    rstream->hdr_offset + rstream->mail_size == offset)

+		return istream_raw_mbox_next(stream, (uoff_t)-1);

 	if (offset == rstream->from_offset && rstream->seeked) {

 		/* back to beginning of current message */

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/istream-raw-mbox.h

--- a/src/lib-storage/index/mbox/istream-raw-mbox.h	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/istream-raw-mbox.h	Wed Nov 19 17:37:57 2014 +0900

@@ -9,15 +9,18 @@

 /* Return offset to beginning of the "\nFrom"-line. */

 uoff_t istream_raw_mbox_get_start_offset(struct istream *stream);

 /* Return offset to beginning of the headers. */

-uoff_t istream_raw_mbox_get_header_offset(struct istream *stream);

+int istream_raw_mbox_get_header_offset(struct istream *stream,

+				       uoff_t *hdr_offset_r);

 /* Return offset to beginning of the body. */

-uoff_t istream_raw_mbox_get_body_offset(struct istream *stream);

+int istream_raw_mbox_get_body_offset(struct istream *stream,

+				     uoff_t *body_offset_r);

 /* Return the number of bytes in the body of this message. If

    expected_body_size isn't (uoff_t)-1, we'll use it as potentially valid body

    size to avoid actually reading through the whole message. */

-uoff_t istream_raw_mbox_get_body_size(struct istream *stream,

-				      uoff_t expected_body_size);

+int istream_raw_mbox_get_body_size(struct istream *stream,

+				   uoff_t expected_body_size,

+				   uoff_t *body_size_r);

 /* Return received time of current message, or (time_t)-1 if the timestamp is

    broken. */

@@ -30,7 +33,7 @@

 /* Jump to next message. If expected_body_size isn't (uoff_t)-1, we'll use it

    as potentially valid body size. */

-void istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size);

+int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size);

 /* Seek to message at given offset. offset must point to beginning of

    "\nFrom ", or 0 for beginning of file. Returns -1 if it offset doesn't

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-mail.c

--- a/src/lib-storage/index/mbox/mbox-mail.c	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/mbox-mail.c	Wed Nov 19 17:37:57 2014 +0900

@@ -304,10 +304,10 @@

 		return -1;

 	/* our header size varies, so don't do any caching */

-	body_offset = istream_raw_mbox_get_body_offset(mbox->mbox_stream);

-	if (body_offset == (uoff_t)-1) {

+	if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {

 		mail_storage_set_critical(_mail->box->storage,

-					  "Couldn't get mbox size");

+			"mbox %s: Couldn't get body offset for uid=%u",

+			mailbox_get_path(&mbox->box), mail->mail.mail.uid);

 		return -1;

 	}

@@ -319,8 +319,13 @@

 		body_size = (uoff_t)-1;

 	/* verify that the calculated body size is correct */

-	body_size = istream_raw_mbox_get_body_size(mbox->mbox_stream,

-						   body_size);

+	if (istream_raw_mbox_get_body_size(mbox->mbox_stream,

+					   body_size, &body_size) < 0) {

+		mail_storage_set_critical(_mail->box->storage,

+			"mbox %s: Couldn't get body size for uid=%u",

+			mailbox_get_path(&mbox->box), mail->mail.mail.uid);

+		return -1;

+	}

 	data->physical_size = hdr_size.physical_size + body_size;

 	*size_r = data->physical_size;

@@ -352,7 +357,12 @@

 	}

 	raw_stream = mbox->mbox_stream;

-	hdr_offset = istream_raw_mbox_get_header_offset(raw_stream);

+	if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {

+		mail_storage_set_critical(mbox->box.storage,

+			"mbox %s: Couldn't get header offset for uid=%u",

+			mailbox_get_path(&mbox->box), mail->mail.mail.uid);

+		return -1;

+	}

 	i_stream_seek(raw_stream, hdr_offset);

 	if (next_offset != (uoff_t)-1)

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-parse.c

--- a/src/lib-storage/index/mbox/mbox-sync-parse.c	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/mbox-sync-parse.c	Wed Nov 19 17:37:57 2014 +0900

@@ -453,8 +453,8 @@

 	return strcasecmp(key, func->header);

 }

-void mbox_sync_parse_next_mail(struct istream *input,

-			       struct mbox_sync_mail_context *ctx)

+int mbox_sync_parse_next_mail(struct istream *input,

+			      struct mbox_sync_mail_context *ctx)

 {

 	struct mbox_sync_context *sync_ctx = ctx->sync_ctx;

 	struct message_header_parser_ctx *hdr_ctx;

@@ -545,6 +545,12 @@

 	}

 	ctx->body_offset = input->v_offset;

+	if (input->stream_errno != 0) {

+		mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s",

+			i_stream_get_name(input), i_stream_get_error(input));

+		return -1;

+	}

+	return 0;

 }

 bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox,

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-private.h

--- a/src/lib-storage/index/mbox/mbox-sync-private.h	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/mbox-sync-private.h	Wed Nov 19 17:37:57 2014 +0900

@@ -158,8 +158,8 @@

 void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,

 			    const char *fmt, ...) ATTR_FORMAT(2, 3);

-void mbox_sync_parse_next_mail(struct istream *input,

-			       struct mbox_sync_mail_context *ctx);

+int mbox_sync_parse_next_mail(struct istream *input,

+			      struct mbox_sync_mail_context *ctx);

 bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox,

 				struct mail_index_view *view, uint32_t seq);

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-rewrite.c

--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c	Wed Nov 19 17:37:57 2014 +0900

@@ -318,11 +318,11 @@

 	return 1;

 }

-static void mbox_sync_read_next(struct mbox_sync_context *sync_ctx,

-				struct mbox_sync_mail_context *mail_ctx,

-				struct mbox_sync_mail *mails,

-				uint32_t seq, uint32_t idx,

-				uoff_t expunged_space)

+static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx,

+			       struct mbox_sync_mail_context *mail_ctx,

+			       struct mbox_sync_mail *mails,

+			       uint32_t seq, uint32_t idx,

+			       uoff_t expunged_space)

 {

 	unsigned int first_mail_expunge_extra;

 	uint32_t orig_next_uid;

@@ -332,8 +332,12 @@

 	mail_ctx->seq = seq;

 	mail_ctx->header = sync_ctx->header;

-	mail_ctx->mail.offset =

-		istream_raw_mbox_get_header_offset(sync_ctx->input);

+	if (istream_raw_mbox_get_header_offset(sync_ctx->input,

+					       &mail_ctx->mail.offset) < 0) {

+		mbox_sync_set_critical(sync_ctx,

+			"Couldn't get header offset for seq=%u", seq);

+		return -1;

+	}

 	mail_ctx->mail.body_size = mails[idx].body_size;

 	orig_next_uid = sync_ctx->next_uid;

@@ -361,7 +365,8 @@

 		mails[idx].from_offset += first_mail_expunge_extra;

 	}

-	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);

+	if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0)

+		return -1;

 	i_assert(mail_ctx->mail.pseudo == mails[idx].pseudo);

 	/* set next_uid back before updating the headers. this is important

@@ -381,6 +386,7 @@

 		if (mail_ctx->have_eoh)

 			str_append_c(mail_ctx->header, '\n');

 	}

+	return 0;

 }

 static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,

@@ -398,8 +404,9 @@

 		if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0)

 			return -1;

-		mbox_sync_read_next(sync_ctx, &new_mail_ctx, mails, seq, idx,

-				    expunged_space);

+		if (mbox_sync_read_next(sync_ctx, &new_mail_ctx, mails, seq, idx,

+					expunged_space) < 0)

+			return -1;

 		mail_ctx = &new_mail_ctx;

 	} else {

 		i_assert(seq == mail_ctx->seq);

diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync.c

--- a/src/lib-storage/index/mbox/mbox-sync.c	Wed Nov 19 17:37:18 2014 +0900

+++ b/src/lib-storage/index/mbox/mbox-sync.c	Wed Nov 19 17:37:57 2014 +0900

@@ -131,8 +131,10 @@

 mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,

 			 struct mbox_sync_mail_context *mail_ctx)

 {

+	uoff_t offset;

+

 	/* get EOF */

-	(void)istream_raw_mbox_get_header_offset(sync_ctx->input);

+	(void)istream_raw_mbox_get_header_offset(sync_ctx->input, &offset);

 	if (istream_raw_mbox_is_eof(sync_ctx->input))

 		return 0;

@@ -144,19 +146,27 @@

 	mail_ctx->mail.from_offset =

 		istream_raw_mbox_get_start_offset(sync_ctx->input);

-	mail_ctx->mail.offset =

-		istream_raw_mbox_get_header_offset(sync_ctx->input);

+	if (istream_raw_mbox_get_header_offset(sync_ctx->input, &mail_ctx->mail.offset) < 0) {

+		mbox_sync_set_critical(sync_ctx,

+			"Couldn't get header offset for seq=%u", mail_ctx->seq);

+		return -1;

+	}

-	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);

+	if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0)

+		return -1;

+	if (istream_raw_mbox_is_corrupted(sync_ctx->input))

+		return -1;

+

 	i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||

 		 sync_ctx->input->eof);

-	if (istream_raw_mbox_is_corrupted(sync_ctx->input))

+	if (istream_raw_mbox_get_body_size(sync_ctx->input,

+					   mail_ctx->content_length,

+					   &mail_ctx->mail.body_size) < 0) {

+		mbox_sync_set_critical(sync_ctx,

+			"Couldn't get body size for seq=%u", mail_ctx->seq);

 		return -1;

-

-	mail_ctx->mail.body_size =

-		istream_raw_mbox_get_body_size(sync_ctx->input,

-					       mail_ctx->content_length);

+	}

 	i_assert(mail_ctx->mail.body_size < OFF_T_MAX);

 	if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&

@@ -810,7 +820,7 @@

 mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)

 {

 	struct mbox_mailbox *mbox = sync_ctx->mbox;

-	uoff_t old_offset;

+	uoff_t old_offset, offset;

 	uint32_t uid;

 	int ret;

         bool deleted;

@@ -864,7 +874,11 @@

         sync_ctx->idx_seq = seq;

 	sync_ctx->dest_first_mail = sync_ctx->seq == 0;

-        (void)istream_raw_mbox_get_body_offset(sync_ctx->input);

+	if (istream_raw_mbox_get_body_offset(sync_ctx->input, &offset) < 0) {

+		mbox_sync_set_critical(sync_ctx,

+			"Message body offset lookup failed");

+		return -1;

+	}

 	return 1;

 }

@@ -1149,8 +1163,9 @@

 			sync_ctx->idx_seq++;

 		}

-		istream_raw_mbox_next(sync_ctx->input,

-				      mail_ctx->mail.body_size);

+		if (istream_raw_mbox_next(sync_ctx->input,

+					  mail_ctx->mail.body_size) < 0)

+			return -1;

 		offset = istream_raw_mbox_get_start_offset(sync_ctx->input);

 		if (sync_ctx->need_space_seq != 0) {