diff -up cups-1.3.4/pdftops/Stream.cxx.CVE-2007-4352,5392,5393 cups-1.3.4/pdftops/Stream.cxx

--- cups-1.3.4/pdftops/Stream.cxx.CVE-2007-4352,5392,5393	2007-07-30 20:53:15.000000000 +0100

+++ cups-1.3.4/pdftops/Stream.cxx	2007-11-07 21:12:19.000000000 +0000

@@ -1274,23 +1274,26 @@ CCITTFaxStream::CCITTFaxStream(Stream *s

   columns = columnsA;

   if (columns < 1) {

     columns = 1;

-  }

-  if (columns + 4 <= 0) {

-    columns = INT_MAX - 4;

+  } else if (columns > INT_MAX - 2) {

+    columns = INT_MAX - 2;

   }

   rows = rowsA;

   endOfBlock = endOfBlockA;

   black = blackA;

-  refLine = (short *)gmallocn(columns + 4, sizeof(short));

-  codingLine = (short *)gmallocn(columns + 3, sizeof(short));

+  // 0 <= codingLine[0] < codingLine[1] < ... < codingLine[n] = columns

+  // ---> max codingLine size = columns + 1

+  // refLine has one extra guard entry at the end

+  // ---> max refLine size = columns + 2

+  codingLine = (int *)gmallocn(columns + 1, sizeof(int));

+  refLine = (int *)gmallocn(columns + 2, sizeof(int));

   eof = gFalse;

   row = 0;

   nextLine2D = encoding < 0;

   inputBits = 0;

-  codingLine[0] = 0;

-  codingLine[1] = refLine[2] = columns;

-  a0 = 1;

+  codingLine[0] = columns;

+  a0i = 1;

+  outputBits = 0;

   buf = EOF;

 }

@@ -1309,9 +1312,9 @@ void CCITTFaxStream::reset() {

   row = 0;

   nextLine2D = encoding < 0;

   inputBits = 0;

-  codingLine[0] = 0;

-  codingLine[1] = refLine[2] = columns;

-  a0 = 1;

+  codingLine[0] = columns;

+  a0i = 1;

+  outputBits = 0;

   buf = EOF;

   // skip any initial zero bits and end-of-line marker, and get the 2D

@@ -1328,163 +1331,228 @@ void CCITTFaxStream::reset() {

   }

 }

+inline void CCITTFaxStream::addPixels(int a1, int blackPixels) {

+  if (a1 > codingLine[a0i]) {

+    if (a1 > columns) {

+      error(getPos(), "CCITTFax row is wrong length (%d)", a1);

+      err = gTrue;

+      a1 = columns;

+    }

+    if ((a0i & 1) ^ blackPixels) {

+      ++a0i;

+    }

+    codingLine[a0i] = a1;

+  }

+}

+

+inline void CCITTFaxStream::addPixelsNeg(int a1, int blackPixels) {

+  if (a1 > codingLine[a0i]) {

+    if (a1 > columns) {

+      error(getPos(), "CCITTFax row is wrong length (%d)", a1);

+      err = gTrue;

+      a1 = columns;

+    }

+    if ((a0i & 1) ^ blackPixels) {

+      ++a0i;

+    }

+    codingLine[a0i] = a1;

+  } else if (a1 < codingLine[a0i]) {

+    if (a1 < 0) {

+      error(getPos(), "Invalid CCITTFax code");

+      err = gTrue;

+      a1 = 0;

+    }

+    while (a0i > 0 && a1 <= codingLine[a0i - 1]) {

+      --a0i;

+    }

+    codingLine[a0i] = a1;

+  }

+}

+

+

 int CCITTFaxStream::lookChar() {

   short code1, code2, code3;

-  int a0New;

-  GBool err, gotEOL;

-  int ret;

-  int bits, i;

+  int b1i, blackPixels, i, bits;

+  GBool gotEOL;

-  // if at eof just return EOF

-  if (eof && codingLine[a0] >= columns) {

-    return EOF;

+  if (buf != EOF) {

+    return buf;

   }

   // read the next row

-  err = gFalse;

-  if (codingLine[a0] >= columns) {

+  if (outputBits == 0) {

+

+    // if at eof just return EOF

+    if (eof) {

+      return EOF;

+    }

+

+    err = gFalse;

     // 2-D encoding

     if (nextLine2D) {

       for (i = 0; codingLine[i] < columns; ++i)

 	refLine[i] = codingLine[i];

-      refLine[i] = refLine[i + 1] = columns;

-      b1 = 1;

-      a0New = codingLine[a0 = 0] = 0;

-      do {

+      refLine[i++] = columns;

+      refLine[i] = columns;

+      codingLine[0] = 0;

+      a0i = 0;

+      b1i = 0;

+      blackPixels = 0;

+      // invariant:

+      // refLine[b1i-1] <= codingLine[a0i] < refLine[b1i] < refLine[b1i+1]

+      //                                                             <= columns

+      // exception at left edge:

+      //   codingLine[a0i = 0] = refLine[b1i = 0] = 0 is possible

+      // exception at right edge:

+      //   refLine[b1i] = refLine[b1i+1] = columns is possible

+      while (codingLine[a0i] < columns) {

 	code1 = getTwoDimCode();

 	switch (code1) {

 	case twoDimPass:

-	  if (refLine[b1] < columns) {

-	    a0New = refLine[b1 + 1];

-	    b1 += 2;

+	  addPixels(refLine[b1i + 1], blackPixels);

+	  if (refLine[b1i + 1] < columns) {

+	    b1i += 2;

 	  }

 	  break;

 	case twoDimHoriz:

-	  if ((a0 & 1) == 0) {

-	    code1 = code2 = 0;

+	  code1 = code2 = 0;

+	  if (blackPixels) {

 	    do {

-	      code1 += code3 = getWhiteCode();

+	      code1 += code3 = getBlackCode();

 	    } while (code3 >= 64);

 	    do {

-	      code2 += code3 = getBlackCode();

+	      code2 += code3 = getWhiteCode();

 	    } while (code3 >= 64);

 	  } else {

-	    code1 = code2 = 0;

 	    do {

-	      code1 += code3 = getBlackCode();

+	      code1 += code3 = getWhiteCode();

 	    } while (code3 >= 64);

 	    do {

-	      code2 += code3 = getWhiteCode();

+	      code2 += code3 = getBlackCode();

 	    } while (code3 >= 64);

 	  }

-	  if (code1 > 0 || code2 > 0) {

-	    codingLine[a0 + 1] = a0New + code1;

-	    ++a0;

-	    a0New = codingLine[a0 + 1] = codingLine[a0] + code2;

-	    ++a0;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	  addPixels(codingLine[a0i] + code1, blackPixels);

+	  if (codingLine[a0i] < columns) {

+	    addPixels(codingLine[a0i] + code2, blackPixels ^ 1);

+	  }

+	  while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	    b1i += 2;

 	  }

 	  break;

-	case twoDimVert0:

-	  a0New = codingLine[++a0] = refLine[b1];

-	  if (refLine[b1] < columns) {

-	    ++b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertR3:

+	  addPixels(refLine[b1i] + 3, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    ++b1i;

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

-	case twoDimVertR1:

-	  a0New = codingLine[++a0] = refLine[b1] + 1;

-	  if (refLine[b1] < columns) {

-	    ++b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertR2:

+	  addPixels(refLine[b1i] + 2, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    ++b1i;

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

-	case twoDimVertL1:

-	  if (a0 == 0 || refLine[b1] - 1 > a0New) {

-	    a0New = codingLine[++a0] = refLine[b1] - 1;

-	    --b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertR1:

+	  addPixels(refLine[b1i] + 1, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    ++b1i;

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

-	case twoDimVertR2:

-	  a0New = codingLine[++a0] = refLine[b1] + 2;

-	  if (refLine[b1] < columns) {

-	    ++b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVert0:

+	  addPixels(refLine[b1i], blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    ++b1i;

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

-	case twoDimVertL2:

-	  if (a0 == 0 || refLine[b1] - 2 > a0New) {

-	    a0New = codingLine[++a0] = refLine[b1] - 2;

-	    --b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertL3:

+	  addPixelsNeg(refLine[b1i] - 3, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    if (b1i > 0) {

+	      --b1i;

+	    } else {

+	      ++b1i;

+	    }

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

-	case twoDimVertR3:

-	  a0New = codingLine[++a0] = refLine[b1] + 3;

-	  if (refLine[b1] < columns) {

-	    ++b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertL2:

+	  addPixelsNeg(refLine[b1i] - 2, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    if (b1i > 0) {

+	      --b1i;

+	    } else {

+	      ++b1i;

+	    }

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+  	    }

 	  }

 	  break;

-	case twoDimVertL3:

-	  if (a0 == 0 || refLine[b1] - 3 > a0New) {

-	    a0New = codingLine[++a0] = refLine[b1] - 3;

-	    --b1;

-	    while (refLine[b1] <= codingLine[a0] && refLine[b1] < columns)

-	      b1 += 2;

+	case twoDimVertL1:

+	  addPixelsNeg(refLine[b1i] - 1, blackPixels);

+	  blackPixels ^= 1;

+	  if (codingLine[a0i] < columns) {

+	    if (b1i > 0) {

+	      --b1i;

+	    } else {

+	      ++b1i;

+	    }

+	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {

+	      b1i += 2;

+	    }

 	  }

 	  break;

 	case EOF:

+	  addPixels(columns, 0);

 	  eof = gTrue;

-	  codingLine[a0 = 0] = columns;

-	  return EOF;

+	  break;

 	default:

 	  error(getPos(), "Bad 2D code %04x in CCITTFax stream", code1);

+	  addPixels(columns, 0);

 	  err = gTrue;

 	  break;

 	}

-      } while (codingLine[a0] < columns);

+      }

     // 1-D encoding

     } else {

-      codingLine[a0 = 0] = 0;

-      while (1) {

+      codingLine[0] = 0;

+      a0i = 0;

+      blackPixels = 0;

+      while (codingLine[a0i] < columns) {

 	code1 = 0;

-	do {

-	  code1 += code3 = getWhiteCode();

-	} while (code3 >= 64);

-	codingLine[a0+1] = codingLine[a0] + code1;

-	++a0;

-	if (codingLine[a0] >= columns)

-	  break;

-	code2 = 0;

-	do {

-	  code2 += code3 = getBlackCode();

-	} while (code3 >= 64);

-	codingLine[a0+1] = codingLine[a0] + code2;

-	++a0;

-	if (codingLine[a0] >= columns)

-	  break;

-      }

-    }

-

-    if (codingLine[a0] != columns) {

-      error(getPos(), "CCITTFax row is wrong length (%d)", codingLine[a0]);

-      // force the row to be the correct length

-      while (codingLine[a0] > columns) {

-	--a0;

+	if (blackPixels) {

+	  do {

+	    code1 += code3 = getBlackCode();

+	  } while (code3 >= 64);

+	} else {

+	  do {

+	    code1 += code3 = getWhiteCode();

+	  } while (code3 >= 64);

+	}

+	addPixels(codingLine[a0i] + code1, blackPixels);

+	blackPixels ^= 1;

       }

-      codingLine[++a0] = columns;

-      err = gTrue;

     }

     // byte-align the row

@@ -1545,14 +1613,17 @@ int CCITTFaxStream::lookChar() {

     // this if we know the stream contains end-of-line markers because

     // the "just plow on" technique tends to work better otherwise

     } else if (err && endOfLine) {

-      do {

+      while (1) {

+	code1 = lookBits(13);

 	if (code1 == EOF) {

 	  eof = gTrue;

 	  return EOF;

 	}

+	if ((code1 >> 1) == 0x001) {

+	  break;

+	}

 	eatBits(1);

-	code1 = lookBits(13);

-      } while ((code1 >> 1) != 0x001);

+      }

       eatBits(12); 

       if (encoding > 0) {

 	eatBits(1);

@@ -1560,11 +1631,11 @@ int CCITTFaxStream::lookChar() {

       }

     }

-    a0 = 0;

-    outputBits = codingLine[1] - codingLine[0];

-    if (outputBits == 0) {

-      a0 = 1;

-      outputBits = codingLine[2] - codingLine[1];

+    // set up for output

+    if (codingLine[0] > 0) {

+      outputBits = codingLine[a0i = 0];

+    } else {

+      outputBits = codingLine[a0i = 1];

     }

     ++row;

@@ -1572,39 +1643,43 @@ int CCITTFaxStream::lookChar() {

   // get a byte

   if (outputBits >= 8) {

-    ret = ((a0 & 1) == 0) ? 0xff : 0x00;

-    if ((outputBits -= 8) == 0) {

-      ++a0;

-      if (codingLine[a0] < columns) {

-	outputBits = codingLine[a0 + 1] - codingLine[a0];

-      }

+    buf = (a0i & 1) ? 0x00 : 0xff;

+    outputBits -= 8;

+    if (outputBits == 0 && codingLine[a0i] < columns) {

+      ++a0i;

+      outputBits = codingLine[a0i] - codingLine[a0i - 1];

     }

   } else {

     bits = 8;

-    ret = 0;

+    buf = 0;

     do {

       if (outputBits > bits) {

-	i = bits;

-	bits = 0;

-	if ((a0 & 1) == 0) {

-	  ret |= 0xff >> (8 - i);

+	buf <<= bits;

+	if (!(a0i & 1)) {

+	  buf |= 0xff >> (8 - bits);

 	}

-	outputBits -= i;

+	outputBits -= bits;

+	bits = 0;

       } else {

-	i = outputBits;

-	bits -= outputBits;

-	if ((a0 & 1) == 0) {

-	  ret |= (0xff >> (8 - i)) << bits;

+	buf <<= outputBits;

+	if (!(a0i & 1)) {

+	  buf |= 0xff >> (8 - outputBits);

 	}

+	bits -= outputBits;

 	outputBits = 0;

-	++a0;

-	if (codingLine[a0] < columns) {

-	  outputBits = codingLine[a0 + 1] - codingLine[a0];

+	if (codingLine[a0i] < columns) {

+	  ++a0i;

+	  outputBits = codingLine[a0i] - codingLine[a0i - 1];

+	} else if (bits > 0) {

+	  buf <<= bits;

+	  bits = 0;

 	}

       }

-    } while (bits > 0 && codingLine[a0] < columns);

+    } while (bits);

+  }

+  if (black) {

+    buf ^= 0xff;

   }

-  buf = black ? (ret ^ 0xff) : ret;

   return buf;

 }

@@ -1646,6 +1721,9 @@ short CCITTFaxStream::getWhiteCode() {

   code = 0; // make gcc happy

   if (endOfBlock) {

     code = lookBits(12);

+    if (code == EOF) {

+      return 1;

+    }

     if ((code >> 5) == 0) {

       p = &whiteTab1[code];

     } else {

@@ -1658,6 +1736,9 @@ short CCITTFaxStream::getWhiteCode() {

   } else {

     for (n = 1; n <= 9; ++n) {

       code = lookBits(n);

+      if (code == EOF) {

+	return 1;

+      }

       if (n < 9) {

 	code <<= 9 - n;

       }

@@ -1669,6 +1750,9 @@ short CCITTFaxStream::getWhiteCode() {

     }

     for (n = 11; n <= 12; ++n) {

       code = lookBits(n);

+      if (code == EOF) {

+	return 1;

+      }

       if (n < 12) {

 	code <<= 12 - n;

       }

@@ -1694,9 +1778,12 @@ short CCITTFaxStream::getBlackCode() {

   code = 0; // make gcc happy

   if (endOfBlock) {

     code = lookBits(13);

+    if (code == EOF) {

+      return 1;

+    }

     if ((code >> 7) == 0) {

       p = &blackTab1[code];

-    } else if ((code >> 9) == 0) {

+    } else if ((code >> 9) == 0 && (code >> 7) != 0) {

       p = &blackTab2[(code >> 1) - 64];

     } else {

       p = &blackTab3[code >> 7];

@@ -1708,6 +1795,9 @@ short CCITTFaxStream::getBlackCode() {

   } else {

     for (n = 2; n <= 6; ++n) {

       code = lookBits(n);

+      if (code == EOF) {

+	return 1;

+      }

       if (n < 6) {

 	code <<= 6 - n;

       }

@@ -1719,6 +1809,9 @@ short CCITTFaxStream::getBlackCode() {

     }

     for (n = 7; n <= 12; ++n) {

       code = lookBits(n);

+      if (code == EOF) {

+	return 1;

+      }

       if (n < 12) {

 	code <<= 12 - n;

       }

@@ -1732,6 +1825,9 @@ short CCITTFaxStream::getBlackCode() {

     }

     for (n = 10; n <= 13; ++n) {

       code = lookBits(n);

+      if (code == EOF) {

+	return 1;

+      }

       if (n < 13) {

 	code <<= 13 - n;

       }

@@ -1954,6 +2050,12 @@ void DCTStream::reset() {

     // allocate a buffer for the whole image

     bufWidth = ((width + mcuWidth - 1) / mcuWidth) * mcuWidth;

     bufHeight = ((height + mcuHeight - 1) / mcuHeight) * mcuHeight;

+    if (bufWidth <= 0 || bufHeight <= 0 ||

+	bufWidth > INT_MAX / bufWidth / (int)sizeof(int)) {

+      error(getPos(), "Invalid image size in DCT stream");

+      y = height;

+      return;

+    }

     for (i = 0; i < numComps; ++i) {

       frameBuf[i] = (int *)gmallocn(bufWidth * bufHeight, sizeof(int));

       memset(frameBuf[i], 0, bufWidth * bufHeight * sizeof(int));

@@ -3015,6 +3117,11 @@ GBool DCTStream::readScanInfo() {

   }

   scanInfo.firstCoeff = str->getChar();

   scanInfo.lastCoeff = str->getChar();

+  if (scanInfo.firstCoeff < 0 || scanInfo.lastCoeff > 63 ||

+      scanInfo.firstCoeff > scanInfo.lastCoeff) {

+    error(getPos(), "Bad DCT coefficient numbers in scan info block");

+    return gFalse;

+  }

   c = str->getChar();

   scanInfo.ah = (c >> 4) & 0x0f;

   scanInfo.al = c & 0x0f;

diff -up cups-1.3.4/pdftops/Stream.h.CVE-2007-4352,5392,5393 cups-1.3.4/pdftops/Stream.h

--- cups-1.3.4/pdftops/Stream.h.CVE-2007-4352,5392,5393	2006-02-13 03:08:11.000000000 +0000

+++ cups-1.3.4/pdftops/Stream.h	2007-11-07 21:12:19.000000000 +0000

@@ -519,13 +519,15 @@ private:

   int row;			// current row

   int inputBuf;			// input buffer

   int inputBits;		// number of bits in input buffer

-  short *refLine;		// reference line changing elements

-  int b1;			// index into refLine

-  short *codingLine;		// coding line changing elements

-  int a0;			// index into codingLine

+  int *codingLine;		// coding line changing elements

+  int *refLine;			// reference line changing elements

+  int a0i;			// index into codingLine

+  GBool err;			// error on current line

   int outputBits;		// remaining ouput bits

   int buf;			// character buffer

+  void addPixels(int a1, int black);

+  void addPixelsNeg(int a1, int black);

   short getTwoDimCode();

   short getWhiteCode();

   short getBlackCode();