/* Tokenizer implementation */

#include "Python.h"

#include "pgenheaders.h"

#include <ctype.h>

#include <assert.h>

#include "tokenizer.h"

#include "errcode.h"

#ifndef PGEN

#include "unicodeobject.h"

#include "stringobject.h"

#include "fileobject.h"

#include "codecs.h"

#include "abstract.h"

#include "pydebug.h"

#endif /* PGEN */

extern char *PyOS_Readline(FILE *, FILE *, char *);

/* Return malloc'ed string including trailing \n;

   empty malloc'ed string for EOF;

   NULL if interrupted */

/* Don't ever change this -- it would break the portability of Python code */

#define TABSIZE 8

/* Forward */

static struct tok_state *tok_new(void);

static int tok_nextc(struct tok_state *tok);

static void tok_backup(struct tok_state *tok, int c);

/* Token names */

char *_PyParser_TokenNames[] = {

	"ENDMARKER",

	"NAME",

	"NUMBER",

	"STRING",

	"NEWLINE",

	"INDENT",

	"DEDENT",

	"LPAR",

	"RPAR",

	"LSQB",

	"RSQB",

	"COLON",

	"COMMA",

	"SEMI",

	"PLUS",

	"MINUS",

	"STAR",

	"SLASH",

	"VBAR",

	"AMPER",

	"LESS",

	"GREATER",

	"EQUAL",

	"DOT",

	"PERCENT",

	"BACKQUOTE",

	"LBRACE",

	"RBRACE",

	"EQEQUAL",

	"NOTEQUAL",

	"LESSEQUAL",

	"GREATEREQUAL",

	"TILDE",

	"CIRCUMFLEX",

	"LEFTSHIFT",

	"RIGHTSHIFT",

	"DOUBLESTAR",

	"PLUSEQUAL",

	"MINEQUAL",

	"STAREQUAL",

	"SLASHEQUAL",

	"PERCENTEQUAL",

	"AMPEREQUAL",

	"VBAREQUAL",

	"CIRCUMFLEXEQUAL",

	"LEFTSHIFTEQUAL",

	"RIGHTSHIFTEQUAL",

	"DOUBLESTAREQUAL",

	"DOUBLESLASH",

	"DOUBLESLASHEQUAL",

	"AT",

	/* This table must match the #defines in token.h! */

	"OP",

	"<ERRORTOKEN>",

	"<N_TOKENS>"

};

/* Create and initialize a new tok_state structure */

static struct tok_state *

tok_new(void)

{

	struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(

                                                sizeof(struct tok_state));

	if (tok == NULL)

		return NULL;

	tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;

	tok->done = E_OK;

	tok->fp = NULL;

	tok->tabsize = TABSIZE;

	tok->indent = 0;

	tok->indstack[0] = 0;

	tok->atbol = 1;

	tok->pendin = 0;

	tok->prompt = tok->nextprompt = NULL;

	tok->lineno = 0;

	tok->level = 0;

	tok->filename = NULL;

	tok->altwarning = 0;

	tok->alterror = 0;

	tok->alttabsize = 1;

	tok->altindstack[0] = 0;

	tok->decoding_state = 0;

	tok->decoding_erred = 0;

	tok->read_coding_spec = 0;

	tok->encoding = NULL;

        tok->cont_line = 0;

#ifndef PGEN

	tok->decoding_readline = NULL;

	tok->decoding_buffer = NULL;

#endif

	return tok;

}

#ifdef PGEN

static char *

decoding_fgets(char *s, int size, struct tok_state *tok)

{

	return fgets(s, size, tok->fp);

}

static int

decoding_feof(struct tok_state *tok)

{

	return feof(tok->fp);

}

static const char *

decode_str(const char *str, struct tok_state *tok)

{

	return str;

}

#else /* PGEN */

static char *

error_ret(struct tok_state *tok) /* XXX */

{

	tok->decoding_erred = 1;

	if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */

		PyMem_FREE(tok->buf);

	tok->buf = NULL;

	return NULL;		/* as if it were EOF */

}

static char *

new_string(const char *s, Py_ssize_t len)

{

	char* result = (char *)PyMem_MALLOC(len + 1);

	if (result != NULL) {

		memcpy(result, s, len);

		result[len] = '\0';

	}

	return result;

}

static char *

get_normal_name(char *s)	/* for utf-8 and latin-1 */

{

	char buf[13];

	int i;

	for (i = 0; i < 12; i++) {

		int c = s[i];

		if (c == '\0') break;

		else if (c == '_') buf[i] = '-';

		else buf[i] = tolower(c);

	}

	buf[i] = '\0';

	if (strcmp(buf, "utf-8") == 0 ||

	    strncmp(buf, "utf-8-", 6) == 0) return "utf-8";

	else if (strcmp(buf, "latin-1") == 0 ||

		 strcmp(buf, "iso-8859-1") == 0 ||

		 strcmp(buf, "iso-latin-1") == 0 ||

		 strncmp(buf, "latin-1-", 8) == 0 ||

		 strncmp(buf, "iso-8859-1-", 11) == 0 ||

		 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";

	else return s;

}

/* Return the coding spec in S, or NULL if none is found.  */

static char *

get_coding_spec(const char *s, Py_ssize_t size)

{

	Py_ssize_t i;

	/* Coding spec must be in a comment, and that comment must be

         * the only statement on the source code line. */

        for (i = 0; i < size - 6; i++) {

		if (s[i] == '#')

			break;

		if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')

			return NULL;

	}

	for (; i < size - 6; i++) { /* XXX inefficient search */

		const char* t = s + i;

		if (strncmp(t, "coding", 6) == 0) {

			const char* begin = NULL;

			t += 6;

			if (t[0] != ':' && t[0] != '=')

				continue;

			do {

				t++;

			} while (t[0] == '\x20' || t[0] == '\t');

			begin = t;

			while (isalnum(Py_CHARMASK(t[0])) ||

			       t[0] == '-' || t[0] == '_' || t[0] == '.')

				t++;

			if (begin < t) {

				char* r = new_string(begin, t - begin);

				char* q = get_normal_name(r);

				if (r != q) {

					PyMem_FREE(r);

					r = new_string(q, strlen(q));

				}

				return r;

			}

		}

	}

	return NULL;

}

/* Check whether the line contains a coding spec. If it does,

   invoke the set_readline function for the new encoding.

   This function receives the tok_state and the new encoding.

   Return 1 on success, 0 on failure.  */

static int

check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,

		  int set_readline(struct tok_state *, const char *))

{

	char * cs;

	int r = 1;

        if (tok->cont_line)

		/* It's a continuation line, so it can't be a coding spec. */

		return 1;

	cs = get_coding_spec(line, size);

	if (cs != NULL) {

		tok->read_coding_spec = 1;

		if (tok->encoding == NULL) {

			assert(tok->decoding_state == 1); /* raw */

			if (strcmp(cs, "utf-8") == 0 ||

			    strcmp(cs, "iso-8859-1") == 0) {

				tok->encoding = cs;

			} else {

#ifdef Py_USING_UNICODE

				r = set_readline(tok, cs);

				if (r) {

					tok->encoding = cs;

					tok->decoding_state = -1;

				}

				else

					PyMem_FREE(cs);

#else

                                /* Without Unicode support, we cannot

                                   process the coding spec. Since there

                                   won't be any Unicode literals, that

                                   won't matter. */

				PyMem_FREE(cs);

#endif

			}

		} else {	/* then, compare cs with BOM */

			r = (strcmp(tok->encoding, cs) == 0);

			PyMem_FREE(cs);

		}

	}

	if (!r) {

		cs = tok->encoding;

		if (!cs)

			cs = "with BOM";

		PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);

	}

	return r;

}

/* See whether the file starts with a BOM. If it does,

   invoke the set_readline function with the new encoding.

   Return 1 on success, 0 on failure.  */

static int

check_bom(int get_char(struct tok_state *),

	  void unget_char(int, struct tok_state *),

	  int set_readline(struct tok_state *, const char *),

	  struct tok_state *tok)

{

	int ch = get_char(tok);

	tok->decoding_state = 1;

	if (ch == EOF) {

		return 1;

	} else if (ch == 0xEF) {

		ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;

		ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;

#if 0

	/* Disable support for UTF-16 BOMs until a decision

	   is made whether this needs to be supported.  */

	} else if (ch == 0xFE) {

		ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;

		if (!set_readline(tok, "utf-16-be")) return 0;

		tok->decoding_state = -1;

	} else if (ch == 0xFF) {

		ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;

		if (!set_readline(tok, "utf-16-le")) return 0;

		tok->decoding_state = -1;

#endif

	} else {

		unget_char(ch, tok);

		return 1;

	}

	if (tok->encoding != NULL)

		PyMem_FREE(tok->encoding);

	tok->encoding = new_string("utf-8", 5);	/* resulting is in utf-8 */

	return 1;

  NON_BOM:

	/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */

	unget_char(0xFF, tok);	/* XXX this will cause a syntax error */

	return 1;

}

/* Read a line of text from TOK into S, using the stream in TOK.

   Return NULL on failure, else S.

   On entry, tok->decoding_buffer will be one of:

     1) NULL: need to call tok->decoding_readline to get a new line

     2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and

           stored the result in tok->decoding_buffer

     3) PyStringObject *: previous call to fp_readl did not have enough room

           (in the s buffer) to copy entire contents of the line read

           by tok->decoding_readline.  tok->decoding_buffer has the overflow.

           In this case, fp_readl is called in a loop (with an expanded buffer)

           until the buffer ends with a '\n' (or until the end of the file is

           reached): see tok_nextc and its calls to decoding_fgets.

*/

static char *

fp_readl(char *s, int size, struct tok_state *tok)

{

#ifndef Py_USING_UNICODE

	/* In a non-Unicode built, this should never be called. */

	Py_FatalError("fp_readl should not be called in this build.");

	return NULL; /* Keep compiler happy (not reachable) */

#else

	PyObject* utf8 = NULL;

	PyObject* buf = tok->decoding_buffer;

	char *str;

	Py_ssize_t utf8len;

	/* Ask for one less byte so we can terminate it */

	assert(size > 0);

	size--;

	if (buf == NULL) {

		buf = PyObject_CallObject(tok->decoding_readline, NULL);

		if (buf == NULL)

			return error_ret(tok);

	} else {

		tok->decoding_buffer = NULL;

		if (PyString_CheckExact(buf))

			utf8 = buf;

	}

	if (utf8 == NULL) {

		utf8 = PyUnicode_AsUTF8String(buf);

		Py_DECREF(buf);

		if (utf8 == NULL)

			return error_ret(tok);

	}

	str = PyString_AsString(utf8);

	utf8len = PyString_GET_SIZE(utf8);

	if (utf8len > size) {

		tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);

		if (tok->decoding_buffer == NULL) {

			Py_DECREF(utf8);

			return error_ret(tok);

		}

		utf8len = size;

	}

	memcpy(s, str, utf8len);

	s[utf8len] = '\0';

	Py_DECREF(utf8);

	if (utf8len == 0) return NULL; /* EOF */

	return s;

#endif

}

/* Set the readline function for TOK to a StreamReader's

   readline function. The StreamReader is named ENC.

   This function is called from check_bom and check_coding_spec.

   ENC is usually identical to the future value of tok->encoding,

   except for the (currently unsupported) case of UTF-16.

   Return 1 on success, 0 on failure. */

static int

fp_setreadl(struct tok_state *tok, const char* enc)

{

	PyObject *reader, *stream, *readline;

	/* XXX: constify filename argument. */

	stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);

	if (stream == NULL)

		return 0;

	reader = PyCodec_StreamReader(enc, stream, NULL);

	Py_DECREF(stream);

	if (reader == NULL)

		return 0;

	readline = PyObject_GetAttrString(reader, "readline");

	Py_DECREF(reader);

	if (readline == NULL)

		return 0;

	tok->decoding_readline = readline;

	return 1;

}

/* Fetch the next byte from TOK. */

static int fp_getc(struct tok_state *tok) {

	return getc(tok->fp);

}

/* Unfetch the last byte back into TOK.  */

static void fp_ungetc(int c, struct tok_state *tok) {

	ungetc(c, tok->fp);

}

/* Read a line of input from TOK. Determine encoding

   if necessary.  */

static char *

decoding_fgets(char *s, int size, struct tok_state *tok)

{

	char *line = NULL;

	int badchar = 0;

	for (;;) {

		if (tok->decoding_state < 0) {

			/* We already have a codec associated with

			   this input. */

			line = fp_readl(s, size, tok);

			break;

		} else if (tok->decoding_state > 0) {

			/* We want a 'raw' read. */

			line = Py_UniversalNewlineFgets(s, size,

							tok->fp, NULL);

			break;

		} else {

			/* We have not yet determined the encoding.

			   If an encoding is found, use the file-pointer

			   reader functions from now on. */

			if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))

				return error_ret(tok);

			assert(tok->decoding_state != 0);

		}

	}

	if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {

		if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {

			return error_ret(tok);

		}

	}

#ifndef PGEN

	/* The default encoding is ASCII, so make sure we don't have any

           non-ASCII bytes in it. */

	if (line && !tok->encoding) {

		unsigned char *c;

		for (c = (unsigned char *)line; *c; c++)

			if (*c > 127) {

				badchar = *c;

				break;

			}

	}

	if (badchar) {

		char buf[500];

		/* Need to add 1 to the line number, since this line

		   has not been counted, yet.  */

		sprintf(buf,

			"Non-ASCII character '\\x%.2x' "

			"in file %.200s on line %i, "

			"but no encoding declared; "

			"see http://www.python.org/peps/pep-0263.html for details",

			badchar, tok->filename, tok->lineno + 1);

		PyErr_SetString(PyExc_SyntaxError, buf);

		return error_ret(tok);

	}

#endif

	return line;

}

static int

decoding_feof(struct tok_state *tok)

{

	if (tok->decoding_state >= 0) {

		return feof(tok->fp);

	} else {

		PyObject* buf = tok->decoding_buffer;

		if (buf == NULL) {

			buf = PyObject_CallObject(tok->decoding_readline, NULL);

			if (buf == NULL) {

				error_ret(tok);

				return 1;

			} else {

				tok->decoding_buffer = buf;

			}

		}

		return PyObject_Length(buf) == 0;