we are using a custom SC server with the javaViewer and we found what seems to be a bug in the encoding negotiation.
When asked for tight encoding for example, SC will return ZRLE, but the javaViewer does not support ZRLE (it does support Tight though)
Then when asking for Tight, we get ZRLE and it does not work at all...
What we did is quite simple, we just commented the unused cases in the switch, that way if an unknown encoding is requested, it returns FALSE and continues according to the VNC protocol
here is the corrected function we are using. The negociation should now be correct in every case.
Could this modification be included in a future release?
in vncencodemgr.h
Code: Select all
inline BOOL
vncEncodeMgr::SetEncoding(CARD32 encoding,BOOL reinitialize)
{
if (reinitialize)
{
encoding=m_encoding;
m_buffer->CheckBuffer();
}
else m_encoding=encoding;
// Delete the old encoder
if (m_encoder != NULL)
{
if (m_encoder != zrleEncoder)
delete m_encoder;
m_encoder = NULL;
}
// Returns FALSE if the desired encoding cannot be used
switch(encoding)
{
case rfbEncodingRaw:
vnclog.Print(LL_INTINFO, VNCLOG("raw encoder requested\n"));
// Create a RAW encoder
m_encoder = new vncEncoder;
if (m_encoder == NULL)
return FALSE;
break;
//Michael fix : do not give ZRLE (or else) if we ask for Tight (or else)
/* ZRLE default*/
//case rfbEncodingTight:
//case rfbEncodingZlib:
//case rfbEncodingZlibHex:
/* Hextile default*/
//case rfbEncodingUltra:
//case rfbEncodingRRE:
//case rfbEncodingCoRRE:
case rfbEncodingHextile:
vnclog.Print(LL_INTINFO, VNCLOG("Hextile encoder requested\n"));
// Create a CoRRE encoder
m_encoder = new vncEncodeHexT;
if (m_encoder == NULL)
return FALSE;
break;
case rfbEncodingZRLE:
vnclog.Print(LL_INTINFO, VNCLOG("ZRLE encoder requested\n"));
if (!zrleEncoder)
zrleEncoder = new vncEncodeZRLE;
m_encoder = zrleEncoder;
break;
default:
// An unknown encoding was specified
vnclog.Print(LL_INTERR, VNCLOG("unknown encoder requested\n"));
return FALSE;
}
// Initialise it and give it the pixel format
m_encoder->Init();
m_encoder->SetLocalFormat(
m_scrinfo.format,
m_scrinfo.framebufferWidth,
m_scrinfo.framebufferHeight);
if (m_clientfmtset)
if (!m_encoder->SetRemoteFormat(m_clientformat))
{
vnclog.Print(LL_INTERR, VNCLOG("client pixel format is not supported\n"));
return FALSE;
}
if (reinitialize)
{
if (m_encoder != NULL)
{
m_encoder->EnableXCursor(m_use_xcursor);
m_encoder->EnableRichCursor(m_use_richcursor);
m_encoder->SetCompressLevel(m_compresslevel);
m_encoder->SetQualityLevel(m_qualitylevel);
m_encoder->EnableLastRect(m_use_lastrect);
}
}
m_buffer->ClearCache();
m_buffer->ClearBack();
m_encoder->SetSWOffset(m_SWOffsetx,m_SWOffsety);
// Check that the client buffer is compatible
return CheckBuffer();
}
