From 9a861cb3e5e7bd58dc0d4aa549c6d3df25f8b24d Mon Sep 17 00:00:00 2001
From: Marius Kittler <mkittler@suse.de>
Date: Fri, 17 Jun 2022 17:29:20 +0200
Subject: [PATCH] Fix using little-endian VNC server on big-endian

* Fix connecting to a little-endian VNC server from a big-endian
  system
* Use byte swap functions instead of custom code to swap the
  byte order
    * The bswap_32()/bswap_16() functions have the same
      behavior as our custom code on little-endian systems and
      therefore this change should not do anything on
      little-endian systems.
    * The bswap_32()/bswap_16() functions change the byte order
      also on big-endian systems (in contrast to our custom
      code).
* The VNC consoles test has been updated to test both cases
  (byte order swap necassary and not necassary). Before it
  only tested the case of a little-endian VNC server (and was
  therefore only failing on bit-endian systems before this
  change).
* Tested the behavior on x86_64 and also on a Tumbleweed
  ppc64 VM.
* See https://progress.opensuse.org/issues/111608.
---
 ppmclibs/tinycv_impl.cc | 23 +++++++----------------
 t/27-consoles-vnc.t     | 14 +++++++++++++-
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/ppmclibs/tinycv_impl.cc b/ppmclibs/tinycv_impl.cc
index c987127d5d..93f40571e8 100644
--- a/ppmclibs/tinycv_impl.cc
+++ b/ppmclibs/tinycv_impl.cc
@@ -10,6 +10,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <sys/time.h>
+#include <byteswap.h>
 
 #include <algorithm> // std::min
 #include <vector>
@@ -647,13 +648,10 @@ void image_map_raw_data_rgb555(Image* a, const unsigned char* data)
 static uint16_t read_u16(const unsigned char* data, size_t& offset,
     bool do_endian_conversion)
 {
-    uint16_t pixel;
+    uint16_t pixel = *(uint16_t*)(data + offset);
+    offset += 2;
     if (do_endian_conversion) {
-        pixel = data[offset++] * 256;
-        pixel += data[offset++];
-    } else {
-        pixel = data[offset++];
-        pixel += data[offset++] * 256;
+        pixel = bswap_16(pixel);
     }
     return pixel;
 }
@@ -668,17 +666,10 @@ Vec3b VNCInfo::read_pixel(const unsigned char* data, size_t& offset)
     if (bytes_per_pixel == 2) {
         pixel = read_u16(data, offset, do_endian_conversion);
     } else if (bytes_per_pixel == 4) {
+        pixel = *(uint32_t*)(data + offset);
+        offset += 4;
         if (do_endian_conversion) {
-            pixel = data[offset++];
-            pixel <<= 8;
-            pixel |= data[offset++];
-            pixel <<= 8;
-            pixel |= data[offset++];
-            pixel <<= 8;
-            pixel |= data[offset++];
-        } else {
-            pixel = *(uint32_t*)(data + offset);
-            offset += 4;
+            pixel = bswap_32(pixel);
         }
     } else if (bytes_per_pixel == 1) {
         pixel = data[offset++];
diff --git a/t/27-consoles-vnc.t b/t/27-consoles-vnc.t
index 455afc0646..be0731c9d6 100755
--- a/t/27-consoles-vnc.t
+++ b/t/27-consoles-vnc.t
@@ -41,7 +41,6 @@ is_deeply \@printed, ['RFB 003.006', pack('C', 1)], 'protocol version and securi
 
 # ensure endian conversion is setup correctly (despite initially mocking _server_initialization)
 my $machine_is_big_endian = unpack('h*', pack('s', 1)) =~ /01/ ? 1 : 0;
-$c->_do_endian_conversion($machine_is_big_endian);
 
 subtest 'send update request' => sub {
     $c->width(1024);
@@ -161,6 +160,7 @@ subtest 'update framebuffer' => sub {
     ok $logged_in, 'relogin on protocol error';
 
     # test with full data (just one pixel, though) and vncinfo present (defines endianness and chroma subsampling)
+    $c->_do_endian_conversion($machine_is_big_endian);    # assume server is little-endian
     my $vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16);
     my $gray_pixel = pack(CCCC => 31, 37, 41, 0);    # dark prime grey
     my $of_type_raw_with_coordinates_43_47_1_1 = pack(nnnnN => 43, 47, 1, 1, 0);
@@ -181,6 +181,18 @@ subtest 'update framebuffer' => sub {
     throws_ok { $c->update_framebuffer } qr/unsupported update encoding -225/, 'dies on unsupported encoding';
     is $s->mocked_read, undef, 'no more messages left to read after reading unknown encoding';
 
+    # test with full data again, assuming the server is big-endian
+    $c->_do_endian_conversion(!$machine_is_big_endian);    # assume server is big-endian
+    $vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16);
+    $gray_pixel = pack(CCCC => 0, 41, 37, 31);    # dark prime grey
+    $s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_raw_with_coordinates_43_47_1_1, $gray_pixel);
+    $c->_framebuffer(undef)->width(1024)->height(512)->vncinfo($vncinfo);
+    ok $c->update_framebuffer, 'truthy return value for successful pixel update of big-endian server';
+    ($blue, $green, $red) = $c->_framebuffer->get_pixel(43, 47);
+    is $blue, 41, 'pixel data updated in framebuffer (blue, big-endian server)';
+    is $green, 37, 'pixel data updated in framebuffer (green, big-endian server)';
+    is $red, 31, 'pixel data updated in framebuffer (red, big-endian server)';
+
     $c->ikvm(1);
     my $unsupported_ikvm_encoding = pack(nnnnN => 0, 0, 1, 1, 88);
     my $ikvm_specific_data = pack(NN => 0, 9);    # some "prefix" and data length