Support explicit integer base prefixes and reject malformed literals

engine/compiler.c

@@ -279,13 +279,80 @@ static ph7_value *PH7_GenStateInstallNumLiteral(ph7_gen_state *pGen, sxu32 *pIdx
 	*pIdx = nIdx;
 	return pObj;
 }
+/*
+ * Return the base selected by an explicit integer literal prefix
+ * [i.e: 0x/0X, 0b/0B, 0o/0O], defaulting to decimal when no explicit
+ * prefix is present.
+ */
+static sxi32 PH7_IntegerLiteralBase(const SyString *pVal) {
+	const char *zIn = pVal->zString;
+	if(pVal->nByte > 1 && zIn[0] == '0') {
+		if(zIn[1] == 'x' || zIn[1] == 'X') {
+			return 16;
+		} else if(zIn[1] == 'b' || zIn[1] == 'B') {
+			return 2;
+		} else if(zIn[1] == 'o' || zIn[1] == 'O') {
+			return 8;
+		}
+	}
+	return 10;
+}
+/*
+ * Validate and convert an integer literal to a 64-bit signed value.
+ * Unlike PH7_TokenValueToInt64(), which is also used by string casts, the
+ * literal base is selected only by an explicit prefix; see
+ * PH7_IntegerLiteralBase().
+ * Return SXRET_OK on success, SXERR_EMPTY when a base prefix is not followed
+ * by any digit, SXERR_SYNTAX on an invalid digit or character for the base,
+ * and SXERR_INVALID on a malformed literal [i.e: 10x41].
+ */
+static sxi32 PH7_IntegerLiteralToInt64(SyString *pVal, sxi64 *pOutVal) {
+	const char *zIn = pVal->zString;
+	sxu32 nLen = pVal->nByte;
+	sxu32 i;
+	sxi32 nBase = PH7_IntegerLiteralBase(pVal);
+	if(nBase == 10) {
+		for(i = 0; i < nLen; i++) {
+			if(!SyisDigit(zIn[i])) {
+				/* Prefix after a leading digit [i.e: 10x41, 00x41] */
+				return SXERR_INVALID;
+			}
+		}
+		SyStrToInt64(zIn, nLen, pOutVal, 0);
+		return SXRET_OK;
+	}
+	if(nLen == 2) {
+		/* Missing digits after the base prefix [i.e: 0x, 0b, 0o] */
+		return SXERR_EMPTY;
+	}
+	for(i = 2; i < nLen; i++) {
+		sxi32 c = zIn[i];
+		if((nBase == 16 && !SyisHex(c)) || (nBase == 8 && (c < '0' || c > '7')) ||
+				(nBase == 2 && c != '0' && c != '1')) {
+			/* Defensive validation for non-lexer token sources: the lexer
+			 * normally stops at the first character that is invalid for the
+			 * selected base. */
+			return SXERR_SYNTAX;
+		}
+	}
+	if(nBase == 16) {
+		SyHexStrToInt64(&zIn[2], nLen - 2, pOutVal, 0);
+	} else if(nBase == 8) {
+		/* SyOctalStrToInt64() does not bypass the prefix, so pass digits only */
+		SyOctalStrToInt64(&zIn[2], nLen - 2, pOutVal, 0);
+	} else {
+		SyBinaryStrToInt64(&zIn[2], nLen - 2, pOutVal, 0);
+	}
+	return SXRET_OK;
+}
 /*
  * Compile a numeric [i.e: integer or real] literal.
  * Notes on the integer type.
  *  Integers can be specified in decimal (base 10), hexadecimal (base 16), octal (base 8)
  *  or binary (base 2) notation, optionally preceded by a sign (- or +).
- *  To use octal notation, precede the number with a 0 (zero). To use hexadecimal
+ *  Octal notation requires the explicit 0o prefix. To use hexadecimal
- *  notation precede the number with 0x. To use binary notation precede the number with 0b.
+ *  notation precede the number with 0x. To use binary notation precede the
+ *  number with 0b. The base prefix is case-insensitive.
  * Symisc eXtension to the integer type.
  *  PH7 introduced platform-independant 64-bit integer unlike the standard PHP engine
  *  where the size of an integer is platform-dependent.That is,the size of an integer
@@ -300,7 +367,20 @@ static sxi32 PH7_CompileNumLiteral(ph7_gen_state *pGen, sxi32 iCompileFlag) {
 	if(pToken->nType & PH7_TK_INTEGER) {
 		ph7_value *pObj;
 		sxi64 iValue;
-		iValue = PH7_TokenValueToInt64(&pToken->sData);
+		sxi32 rc;
+		rc = PH7_IntegerLiteralToInt64(&pToken->sData, &iValue);
+		if(rc != SXRET_OK) {
+			if(rc == SXERR_EMPTY) {
+				PH7_GenCompileError(pGen, E_ERROR, pToken->nLine,
+									"Invalid integer literal '%z': missing digits after base prefix", &pToken->sData);
+			} else if(rc == SXERR_SYNTAX) {
+				PH7_GenCompileError(pGen, E_ERROR, pToken->nLine,
+									"Invalid integer literal '%z': invalid digit for base", &pToken->sData);
+			} else {
+				PH7_GenCompileError(pGen, E_ERROR, pToken->nLine, "Malformed integer literal '%z'", &pToken->sData);
+			}
+			return SXERR_ABORT;
+		}
 		pObj = PH7_GenStateInstallNumLiteral(&(*pGen), &nIdx);
 		if(pObj == 0) {
 			SXUNUSED(iCompileFlag); /* cc warning */
@@ -5083,7 +5163,38 @@ static sxi32 PH7_CompileExpr(
 		}
 	}
 	if(pEnd > pGen->pIn) {
-		SyToken *pTmp = pGen->pEnd;
+		SyToken *pTmp, *pToken;
+		/* Reject integer literals directly followed by an identifier character
+		 * [i.e: 0b10foo, 0x41G] before the expression parser folds the tokens.
+		 * A flagged literal is not a malformed tail when the following character
+		 * starts a keyword or an alphabetic operator such as 'is'
+		 * [i.e: 65is int]; the lexer leaves that character for the next token. */
+		for(pToken = pGen->pIn; pToken < pEnd; pToken++) {
+			SyToken *pNext;
+			sxi32 nBase, c;
+			if((pToken->nType & PH7_TK_INTEGER) == 0 || pToken->pUserData == 0) {
+				continue;
+			}
+			pNext = pToken + 1;
+			if(pNext < pEnd && (pNext->nType & (PH7_TK_KEYWORD | PH7_TK_OP))) {
+				/* The flagged character belongs to a keyword or an alphabetic
+				 * operator token, not to the integer literal. */
+				continue;
+			}
+			nBase = PH7_IntegerLiteralBase(&pToken->sData);
+			c = SX_PTR_TO_INT(pToken->pUserData);
+			if(nBase != 10 && pToken->sData.nByte == 2) {
+				/* The base prefix has no digits yet [i.e: 0b2, 0o8, 0xG] */
+				PH7_GenCompileError(pGen, E_ERROR, pToken->nLine,
+									"Invalid %s '%c' after base prefix '%z'",
+									SyisDigit(c) ? "digit" : "character", c, &pToken->sData);
+			} else {
+				PH7_GenCompileError(pGen, E_ERROR, pToken->nLine,
+									"Invalid %s '%c' after integer literal '%z'",
+									SyisDigit(c) ? "digit" : "character", c, &pToken->sData);
+			}
+		}
+		pTmp = pGen->pEnd;
 		/* Swap delimiter */
 		pGen->pEnd = pEnd;
 		/* Try to get an expression tree */

engine/lexer.c

@@ -176,8 +176,24 @@ static sxi32 TokenizeAerScript(SyStream *pStream, SyToken *pToken, void *pUserDa
 					while(pStream->zText < pStream->zEnd && (pStream->zText[0] == '0' || pStream->zText[0] == '1')) {
 						pStream->zText++;
 					}
+				} else if(c == 'o' || c == 'O') {
+					/* Octal digit stream */
+					pStream->zText++;
+					while(pStream->zText < pStream->zEnd && pStream->zText[0] >= '0' && pStream->zText[0] <= '7') {
+						pStream->zText++;
+					}
 				}
 			}
+			if((pToken->nType & PH7_TK_INTEGER) && pStream->zText < pStream->zEnd && pStream->zText[0] < 0xc0 &&
+					(SyisAlphaNum(pStream->zText[0]) || pStream->zText[0] == '_')) {
+				/* Flag an integer literal followed by identifier characters [i.e: 0b102,
+				 * 0x41G, 0b10foo]. The offending character is left in the stream and its
+				 * code is stored in pUserData, which numeric tokens do not use otherwise,
+				 * so the compiler can reject the literal with a precise diagnostic. A
+				 * following keyword or alphabetic operator [i.e: 65is int] is not part
+				 * of the literal. */
+				pToken->pUserData = SX_INT_TO_PTR(pStream->zText[0]);
+			}
 			/* Record token length */
 			pStr->nByte = (sxu32)((const char *)pStream->zText - pStr->zString);
 			return SXRET_OK;