diff -r 8df58d8c99e8 -r b3ffff030d5c core/src/lexer.cpp
--- a/core/src/lexer.cpp	Tue Oct 26 15:36:30 2010 +0100
+++ b/core/src/lexer.cpp	Thu Oct 28 16:54:54 2010 +0100
@@ -12,6 +12,8 @@
 
 #include "lexer.h"
 
+const TInt KMaxReadLength = 512;
+
 
 //
 // TToken
@@ -52,6 +54,231 @@
 
 
 //
+// CLex - A cut down version of the TLex API that supports reading data incrementally from an RIoReadHandle.
+//
+
+class CLex : public CBase
+	{
+public:
+	static CLex* NewL();
+	~CLex();
+	void Set(const TDesC& aDes);
+	void Set(RIoReadHandle& aHandle);
+	void Purge();
+	void SkipToEnd();
+	TBool EosL();
+	void Mark();
+	void IncL(TInt aNumber);
+	TChar GetL();
+	TChar Peek() const;
+	TPtrC RemainderL(TInt aMinLength);
+	void UnGet();
+	TInt Offset() const;
+	TInt MarkedOffset() const;
+	TPtrC MarkedToken() const;
+	const TUint16* Ptr() const;
+private:
+	CLex();
+	void DoReadL();
+private:
+	TLex iLex;
+	TInt iMarkedOffset;
+	TPtrC iLexPtr;
+	RBuf iLexBuf;
+	TBuf<KMaxReadLength> iReadBuf;
+	RIoReadHandle iHandle;
+	TBool iUsingHandle;
+	TBool iEos;
+	};
+
+CLex* CLex::NewL()
+	{
+	return new(ELeave) CLex();
+	}
+
+CLex::CLex() : iLexPtr(NULL, 0)
+	{
+	}
+
+CLex::~CLex()
+	{
+	iLexBuf.Close();
+	}
+
+void CLex::Set(const TDesC& aDes)
+	{
+	iEos = EFalse;
+	iUsingHandle = EFalse;
+	iMarkedOffset = 0;
+	iLexPtr.Set(aDes);
+	iLex = iLexPtr;
+	}
+
+void CLex::Set(RIoReadHandle& aHandle)
+	{
+	iEos = EFalse;
+	iUsingHandle = ETrue;
+	iMarkedOffset = 0;
+	iHandle = aHandle;
+	iHandle.SetReadMode(RIoReadHandle::EOneOrMore);
+	iLexBuf.Zero();
+	iLex = iLexBuf;
+	}
+
+void CLex::Purge()
+	{
+	if (iUsingHandle)
+		{
+		iLexBuf.Delete(0, iLex.Offset());
+		iLex = iLexBuf;
+		}
+
+	iMarkedOffset = 0;
+	}
+
+void CLex::SkipToEnd()
+	{
+	iEos = ETrue;
+	}
+
+TBool CLex::EosL()
+	{
+	if (!iEos)
+		{
+		if (!iLex.Eos())
+			{
+			// If iLex still has data, then we're definately not at the end of the string.
+			// Do nothing. This test avoids us doing I/O handle reads before draining using the existing data.
+			}
+		else if (iUsingHandle)
+			{
+			DoReadL();
+			}
+		else
+			{
+			iEos = ETrue;
+			}
+		}
+	return iEos;
+	}
+
+void CLex::Mark()
+	{
+	iMarkedOffset = iLex.Offset();
+	}
+
+void CLex::IncL(TInt aNumber)
+	{
+	if (iUsingHandle)
+		{
+		while (!iEos && (iLex.Remainder().Length() < aNumber))
+			{
+			DoReadL();
+			}
+		}
+
+	iLex.Inc(aNumber);
+	}
+
+TChar CLex::GetL()
+	{
+	if (iUsingHandle && !iEos && (iLex.Remainder().Length() < 1))
+		{
+		DoReadL();
+		}
+	return iLex.Get();
+	}
+
+TChar CLex::Peek() const
+	{
+	return iLex.Peek();
+	}
+
+TPtrC CLex::RemainderL(TInt aMinLength)
+	{
+	if (iUsingHandle)
+		{
+		while (!iEos && (iLex.Remainder().Length() < aMinLength))
+			{
+			DoReadL();
+			}
+		}
+	return iLex.Remainder();
+	}
+
+void CLex::UnGet()
+	{
+	iLex.UnGet();
+	}
+
+TInt CLex::Offset() const
+	{
+	return iLex.Offset();
+	}
+
+TInt CLex::MarkedOffset() const
+	{
+	return iMarkedOffset;
+	}
+
+TPtrC CLex::MarkedToken() const
+	{
+	if (iUsingHandle)
+		{
+		return TPtrC(iReadBuf.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset);
+		}
+	else
+		{
+		return TPtrC(iLexPtr.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset);
+		}
+	}
+
+const TUint16* CLex::Ptr() const
+	{
+	if (iUsingHandle)
+		{
+		return iLexBuf.Ptr();
+		}
+	else
+		{
+		return iLexPtr.Ptr();
+		}
+
+	}
+
+void CLex::DoReadL()
+	{
+	ASSERT(iUsingHandle);
+
+	if (!iEos)
+		{
+		if (iReadBuf.Length() == 0) // iReadBuf may contain data if a realloc of iLexBuf failed previously.
+			{
+			TInt err = iHandle.Read(iReadBuf);
+			if (err == KErrEof)
+				{
+				iEos = ETrue;
+				}
+			else
+				{
+				User::LeaveIfError(err);
+				}
+			}
+
+		TInt offset = iLex.Offset();
+		if ((iLexBuf.MaxLength() - iLexBuf.Length()) < iReadBuf.Length())
+			{
+			iLexBuf.ReAllocL(iLexBuf.Length() + iReadBuf.Length());
+			}
+		iLexBuf.Append(iReadBuf);
+		iReadBuf.Zero();
+		iLex = iLexBuf;
+		iLex.Inc(offset);
+		}
+	}
+
+
+//
 // CReservedLookup
 //
 
@@ -84,6 +311,7 @@
 	void DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString);
 	void Reset();
 	TResult Lookup(const TDesC& aDes);
+	TInt Longest() const;
 private:
 	class TReserved
 		{
@@ -95,6 +323,7 @@
 		};
 private:
 	RArray<TReserved> iList;
+	TInt iLongest;
 	};
 
 CReservedLookup* CReservedLookup::NewL()
@@ -110,6 +339,10 @@
 void CReservedLookup::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString)
 	{
 	User::LeaveIfError(iList.Append(TReserved(aTokenType, aString)));
+	if (aString.Length() > iLongest)
+		{
+		iLongest = aString.Length();
+		}
 	}
 
 CReservedLookup::TResult CReservedLookup::Lookup(const TDesC& aDes)
@@ -133,6 +366,11 @@
 	return result;
 	}
 
+TInt CReservedLookup::Longest() const
+	{
+	return iLongest;
+	}
+
 CReservedLookup::TReserved::TReserved(TToken::TType aType, const TDesC& aString)
 	: iType(aType), iString(aString)
 	{
@@ -180,6 +418,7 @@
 
 CLexer::~CLexer()
 	{
+	delete iLex;
 	delete iReservedLookup;
 	}
 
@@ -190,91 +429,112 @@
 
 void CLexer::Set(const TDesC& aDes, const TChar& aEscapeChar)
 	{
-	iLex = aDes;
+	iLex->Set(aDes);
+	iEscapeChar = aEscapeChar;
+	}
+
+void CLexer::Set(RIoReadHandle& aHandle, const TChar& aEscapeChar)
+	{
+	iLex->Set(aHandle);
 	iEscapeChar = aEscapeChar;
 	}
 
-TToken CLexer::NextToken()
+void CLexer::Purge()
+	{
+	iLex->Purge();
+	}
+
+void CLexer::SkipToEnd()
 	{
-	SkipWhiteSpace();
-	iLex.Mark();
+	iLex->SkipToEnd();
+	}
+
+TToken CLexer::NextTokenL()
+	{
+	SkipWhiteSpaceL();
+	iLex->Mark();
 	TToken::TType type(TToken::ENull);
 
-	while (!iLex.Eos())
+	while (!iLex->EosL())
 		{
 		type = TToken::EString;
-		TChar c = iLex.Get();
+		TChar c = iLex->GetL();
 		if (c == iEscapeChar)
 			{
-			iLex.Get();
+			iLex->GetL();
 			}
 		else if ((c == '\'') && (iBehaviour & EHandleSingleQuotes))
 			{
-			SkipSingleQuotedChars();
+			SkipSingleQuotedCharsL();
 			}
 		else if ((c == '\"') && (iBehaviour & EHandleDoubleQuotes))
 			{
-			SkipDoubleQuotedChars();
+			SkipDoubleQuotedCharsL();
 			}
 		else if ((c == '#') && (iBehaviour & EHandleComments))
 			{
-			if (iLex.MarkedToken().Length() > 1)
+			if (iLex->MarkedToken().Length() > 1)
 				{
-				iLex.UnGet();
+				iLex->UnGet();
 				break;
 				}
 			else
 				{
-				SkipComment();
-				if (iLex.Eos())
+				SkipCommentL();
+				if (iLex->EosL())
 					{
 					type = TToken::ENull;
 					break;
 					}
 				else
 					{
-					iLex.Mark();
+					iLex->Mark();
 					}
 				}
 			}
 		else if (c.IsSpace() && (c != '\n') && (c != '\r'))
 			{
-			iLex.UnGet();
+			iLex->UnGet();
 			break;
 			}
 		else
 			{
-			iLex.UnGet();
-			CReservedLookup::TResult result = iReservedLookup->Lookup(iLex.Remainder());
+			iLex->UnGet();
+			CReservedLookup::TResult result = iReservedLookup->Lookup(iLex->RemainderL(iReservedLookup->Longest()));
 			if (result.iResultType == CReservedLookup::TResult::EMatch)
 				{
-				if (iLex.MarkedToken().Length() > 0)
+				if (iLex->MarkedToken().Length() > 0)
 					{
 					break;
 					}
 				else
 					{
-					iLex.Inc(result.iTokenLength);
+					iLex->IncL(result.iTokenLength);
 					type = result.iTokenType;
 					break;
 					}
 				}
-			iLex.Get();
+			iLex->GetL();
 			}
 		}
 
-	return TToken(type, iLex.MarkedToken(), iLex.MarkedOffset());
+	return TToken(type, iLex->MarkedToken(), iLex->MarkedOffset());
 	}
 
 TInt CLexer::CurrentOffset() const
 	{
-	return iLex.Offset();
+	return iLex->Offset();
 	}
 
-TBool CLexer::More()
+TBool CLexer::MoreL()
 	{
-	SkipWhiteSpace();
-	return !iLex.Eos();
+	SkipWhiteSpaceL();
+	return !iLex->EosL();
+	}
+
+const TUint16* CLexer::Ptr() const
+	{
+	return iLex->Ptr();
 	}
 
 CLexer::CLexer(TUint aBehaviour)
@@ -284,18 +544,19 @@
 
 void CLexer::ConstructL()
 	{
+	iLex = CLex::NewL();
 	iReservedLookup = CReservedLookup::NewL();
 	}
 
-void CLexer::SkipSingleQuotedChars()
+void CLexer::SkipSingleQuotedCharsL()
 	{
-	while (!iLex.Eos())
+	while (!iLex->EosL())
 		{
-		TChar c = iLex.Get();
-		if ((c == iEscapeChar) && !iLex.Eos() && (iLex.Peek() == '\''))
+		TChar c = iLex->GetL();
+		if ((c == iEscapeChar) && !iLex->EosL() && (iLex->Peek() == '\''))
 			{
 			// Allow quoted single quote characters. Note, the is a departure from Bash behaviour, but is in line with Perl and is generally helpful.
-			iLex.Get();
+			iLex->GetL();
 			}
 		else if (c == '\'')
 			{
@@ -304,14 +565,14 @@
 		}
 	}
 
-void CLexer::SkipDoubleQuotedChars()
+void CLexer::SkipDoubleQuotedCharsL()
 	{
-	while (!iLex.Eos())
+	while (!iLex->EosL())
 		{
-		TChar c = iLex.Get();
-		if ((c == iEscapeChar) && !iLex.Eos())
+		TChar c = iLex->GetL();
+		if ((c == iEscapeChar) && !iLex->EosL())
 			{
-			iLex.Get();
+			iLex->GetL();
 			}
 		else if (c == '"')
 			{
@@ -320,27 +581,27 @@
 		}
 	}
 
-void CLexer::SkipComment()
+void CLexer::SkipCommentL()
 	{
-	while (!iLex.Eos())
+	while (!iLex->EosL())
 		{
-		TChar c = iLex.Get();
+		TChar c = iLex->GetL();
 		if ((c == '\n') || (c == '\r'))
 			{
-			iLex.UnGet();
+			iLex->UnGet();
 			break;
 			}
 		}
 	}
 
-void CLexer::SkipWhiteSpace()
+void CLexer::SkipWhiteSpaceL()
 	{
-	while (!iLex.Eos())
+	while (!iLex->EosL())
 		{
-		TChar c = iLex.Get();
+		TChar c = iLex->GetL();
 		if (!c.IsSpace() || (c == '\n') || (c == '\r'))
 			{
-			iLex.UnGet();
+			iLex->UnGet();
 			break;
 			}
 		}