Skip to content

Commit 2973609

Browse files
rikardfalkeborndanmar
rikardfalkeborn
authored and
danmar
committed
Keep prefix in string and char literals (cppcheck-opensource#2272)
Keeping the prefix in the token allows cppcheck to print the correct string and char literals in debug and error messages. To achieve this, move some of the helper functions from token.cpp to utils.h so that checks that look at string and char literals can reuse them. This is a large part of this commit. Note that the only user visible change is that when string and char literals are printed in error messages, the prefix is now included. For example: int f() { return test.substr( 0 , 4 ) == U"Hello" ? 0 : 1 ; }; now prints U"Hello" instead of "Hello" in the error message.
1 parent 3871323 commit 2973609

10 files changed

Lines changed: 98 additions & 63 deletions

‎lib/checkstring.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,9 +327,9 @@ void CheckString::incorrectStringCompareError(const Token *tok, const std::strin
327327

328328
void CheckString::incorrectStringBooleanError(const Token *tok, const std::string& string)
329329
{
330-
const bool charLiteral = string[0] == '\'';
330+
const bool charLiteral = isCharLiteral(string);
331331
const std::string literalType = charLiteral ? "char" : "string";
332-
const std::string result = (string == "\'\\0\'") ? "false" : "true";
332+
const std::string result = getCharLiteral(string) == "\\0" ? "false" : "true";
333333
reportError(tok,
334334
Severity::warning,
335335
charLiteral ? "incorrectCharBooleanError" : "incorrectStringBooleanError",

‎lib/mathlib.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -531,8 +531,8 @@ MathLib::bigint MathLib::toLongNumber(const std::string & str)
531531
return static_cast<bigint>(doubleval);
532532
}
533533

534-
if (str[0] == '\'' && str.size() >= 3U && endsWith(str,'\'')) {
535-
return characterLiteralToLongNumber(str.substr(1,str.size()-2));
534+
if (isCharLiteral(str)) {
535+
return characterLiteralToLongNumber(getCharLiteral(str));
536536
}
537537

538538
if (str[0] == '-') {
@@ -600,8 +600,8 @@ static double FloatHexToDoubleNumber(const std::string& str)
600600

601601
double MathLib::toDoubleNumber(const std::string &str)
602602
{
603-
if (str[0] == '\'' && str.size() >= 3U && endsWith(str,'\''))
604-
return characterLiteralToLongNumber(str.substr(1,str.size()-2));
603+
if (isCharLiteral(str))
604+
return characterLiteralToLongNumber(getCharLiteral(str));
605605
if (isIntHex(str))
606606
return static_cast<double>(toLongNumber(str));
607607
// nullcheck

‎lib/token.cpp‎

Lines changed: 18 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -34,22 +34,6 @@
3434
#include <stack>
3535
#include <utility>
3636

37-
static const std::string literal_prefix[4] = {"u8", "u", "U", "L"};
38-
39-
static bool isStringCharLiteral(const std::string &str, char q)
40-
{
41-
42-
if (!endsWith(str, q))
43-
return false;
44-
if (str[0] == q && str.length() > 1)
45-
return true;
46-
47-
for (const std::string & p: literal_prefix) {
48-
if ((str.length() + 1) > p.length() && (str.compare(0, p.size() + 1, (p + q)) == 0))
49-
return true;
50-
}
51-
return false;
52-
}
5337
const std::list<ValueFlow::Value> TokenImpl::mEmptyValueList;
5438

5539
Token::Token(TokensFrontBack *tokensFrontBack) :
@@ -89,9 +73,9 @@ void Token::update_property_info()
8973
if (!mStr.empty()) {
9074
if (mStr == "true" || mStr == "false")
9175
tokType(eBoolean);
92-
else if (isStringCharLiteral(mStr, '\"'))
76+
else if (isStringLiteral(mStr))
9377
tokType(eString);
94-
else if (isStringCharLiteral(mStr, '\''))
78+
else if (isCharLiteral(mStr))
9579
tokType(eChar);
9680
else if (std::isalpha((unsigned char)mStr[0]) || mStr[0] == '_' || mStr[0] == '$') { // Name
9781
if (mImpl->mVarId)
@@ -168,17 +152,11 @@ void Token::update_property_isStandardType()
168152

169153
void Token::update_property_char_string_literal()
170154
{
171-
if (!(mTokType == Token::eString || mTokType == Token::eChar)) // Token has already been updated
155+
if (mTokType != Token::eString && mTokType != Token::eChar)
172156
return;
173157

174-
for (const std::string & p : literal_prefix) {
175-
if (((mTokType == Token::eString) && mStr.compare(0, p.size() + 1, p + "\"") == 0) ||
176-
((mTokType == Token::eChar) && (mStr.compare(0, p.size() + 1, p + "\'") == 0))) {
177-
mStr = mStr.substr(p.size());
178-
isLong(p != "u8");
179-
break;
180-
}
181-
}
158+
isLong(((mTokType == Token::eString) && isPrefixStringCharLiteral(mStr, '"', "L")) ||
159+
((mTokType == Token::eChar) && isPrefixStringCharLiteral(mStr, '\'', "L")));
182160
}
183161

184162
bool Token::isUpperCaseName() const
@@ -195,15 +173,15 @@ bool Token::isUpperCaseName() const
195173
void Token::concatStr(std::string const& b)
196174
{
197175
mStr.erase(mStr.length() - 1);
198-
mStr.append(b.begin() + 1, b.end());
176+
mStr.append(getStringLiteral(b) + "\"");
199177

200178
update_property_info();
201179
}
202180

203181
std::string Token::strValue() const
204182
{
205183
assert(mTokType == eString);
206-
std::string ret(mStr.substr(1, mStr.length() - 2));
184+
std::string ret(getStringLiteral(mStr));
207185
std::string::size_type pos = 0U;
208186
while ((pos = ret.find('\\', pos)) != std::string::npos) {
209187
ret.erase(pos,1U);
@@ -721,8 +699,9 @@ nonneg int Token::getStrLength(const Token *tok)
721699
assert(tok->mTokType == eString);
722700

723701
int len = 0;
724-
std::string::const_iterator it = tok->str().begin() + 1U;
725-
const std::string::const_iterator end = tok->str().end() - 1U;
702+
const std::string str(getStringLiteral(tok->str()));
703+
std::string::const_iterator it = str.begin();
704+
const std::string::const_iterator end = str.end();
726705

727706
while (it != end) {
728707
if (*it == '\\') {
@@ -747,9 +726,9 @@ nonneg int Token::getStrSize(const Token *tok)
747726
{
748727
assert(tok != nullptr);
749728
assert(tok->tokType() == eString);
750-
const std::string &str = tok->str();
729+
const std::string str(getStringLiteral(tok->str()));
751730
int sizeofstring = 1;
752-
for (int i = 1; i < (int)str.size() - 1; i++) {
731+
for (int i = 0; i < (int)str.size(); i++) {
753732
if (str[i] == '\\')
754733
++i;
755734
++sizeofstring;
@@ -760,9 +739,9 @@ nonneg int Token::getStrSize(const Token *tok)
760739
std::string Token::getCharAt(const Token *tok, MathLib::bigint index)
761740
{
762741
assert(tok != nullptr);
763-
764-
std::string::const_iterator it = tok->str().begin() + 1U;
765-
const std::string::const_iterator end = tok->str().end() - 1U;
742+
std::string str(getStringLiteral(tok->str()));
743+
std::string::const_iterator it = str.begin();
744+
const std::string::const_iterator end = str.end();
766745

767746
while (it != end) {
768747
if (index == 0) {
@@ -1161,9 +1140,7 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro)
11611140
if (isComplex())
11621141
os << "_Complex ";
11631142
if (isLong()) {
1164-
if (mTokType == eString || mTokType == eChar)
1165-
os << "L";
1166-
else
1143+
if (!(mTokType == eString || mTokType == eChar))
11671144
os << "long ";
11681145
}
11691146
}
@@ -1428,8 +1405,8 @@ static std::string stringFromTokenRange(const Token* start, const Token* end)
14281405
for (const Token *tok = start; tok && tok != end; tok = tok->next()) {
14291406
if (tok->isUnsigned())
14301407
ret << "unsigned ";
1431-
if (tok->isLong())
1432-
ret << (tok->isLiteral() ? "L" : "long ");
1408+
if (tok->isLong() && !tok->isLiteral())
1409+
ret << "long ";
14331410
if (tok->originalName().empty() || tok->isUnsigned() || tok->isLong()) {
14341411
ret << tok->str();
14351412
} else

‎lib/tokenize.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2484,7 +2484,7 @@ void Tokenizer::combineStringAndCharLiterals()
24842484
for (Token *tok = list.front();
24852485
tok;
24862486
tok = tok->next()) {
2487-
if (tok->str()[0] != '"')
2487+
if (!isStringLiteral(tok->str()))
24882488
continue;
24892489

24902490
tok->str(simplifyString(tok->str()));
@@ -10637,8 +10637,11 @@ void Tokenizer::simplifyMicrosoftStringFunctions()
1063710637
tok->deleteNext();
1063810638
tok->deleteThis();
1063910639
tok->deleteNext();
10640-
if (!ansi)
10640+
if (!ansi) {
1064110641
tok->isLong(true);
10642+
if (tok->str()[0] != 'L')
10643+
tok->str("L" + tok->str());
10644+
}
1064210645
while (Token::Match(tok->next(), "_T|_TEXT|TEXT ( %char%|%str% )")) {
1064310646
tok->next()->deleteNext();
1064410647
tok->next()->deleteThis();

‎lib/utils.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,56 @@ inline bool endsWith(const std::string &str, const char end[], std::size_t endle
3636
return (str.size() >= endlen) && (str.compare(str.size()-endlen, endlen, end)==0);
3737
}
3838

39+
inline static bool isPrefixStringCharLiteral(const std::string &str, char q, const std::string& p)
40+
{
41+
if (!endsWith(str, q))
42+
return false;
43+
if ((str.length() + 1) > p.length() && (str.compare(0, p.size() + 1, p + q) == 0))
44+
return true;
45+
return false;
46+
}
47+
48+
inline static bool isStringCharLiteral(const std::string &str, char q)
49+
{
50+
for (const std::string & p: {
51+
"", "u8", "u", "U", "L"
52+
}) {
53+
if (isPrefixStringCharLiteral(str, q, p))
54+
return true;
55+
}
56+
return false;
57+
}
58+
59+
inline static bool isStringLiteral(const std::string &str)
60+
{
61+
return isStringCharLiteral(str, '"');
62+
}
63+
64+
inline static bool isCharLiteral(const std::string &str)
65+
{
66+
return isStringCharLiteral(str, '\'');
67+
}
68+
69+
inline static std::string getStringCharLiteral(const std::string &str, char q)
70+
{
71+
const std::size_t quotePos = str.find(q);
72+
return str.substr(quotePos + 1U, str.size() - quotePos - 2U);
73+
}
74+
75+
inline static std::string getStringLiteral(const std::string &str)
76+
{
77+
if (isStringLiteral(str))
78+
return getStringCharLiteral(str, '"');
79+
return "";
80+
}
81+
82+
inline static std::string getCharLiteral(const std::string &str)
83+
{
84+
if (isCharLiteral(str))
85+
return getStringCharLiteral(str, '\'');
86+
return "";
87+
}
88+
3989
inline static const char *getOrdinalText(int i)
4090
{
4191
if (i == 1)

‎test/testmathlib.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ class TestMathLib : public TestFixture {
286286
ASSERT_EQUALS((int)('\x10'), MathLib::toLongNumber("'\\x10'"));
287287
ASSERT_EQUALS((int)('\100'), MathLib::toLongNumber("'\\100'"));
288288
ASSERT_EQUALS((int)('\200'), MathLib::toLongNumber("'\\200'"));
289+
ASSERT_EQUALS((int)(L'A'), MathLib::toLongNumber("L'A'"));
289290
#ifdef __GNUC__
290291
// BEGIN Implementation-specific results
291292
ASSERT_EQUALS((int)('AB'), MathLib::toLongNumber("'AB'"));
@@ -375,6 +376,7 @@ class TestMathLib : public TestFixture {
375376
ASSERT_EQUALS_DOUBLE(0.0, MathLib::toDoubleNumber("-0.0"), 0.000001);
376377
ASSERT_EQUALS_DOUBLE(0.0, MathLib::toDoubleNumber("+0.0"), 0.000001);
377378
ASSERT_EQUALS_DOUBLE('0', MathLib::toDoubleNumber("'0'"), 0.000001);
379+
ASSERT_EQUALS_DOUBLE(L'0', MathLib::toDoubleNumber("L'0'"), 0.000001);
378380

379381
ASSERT_EQUALS_DOUBLE(192, MathLib::toDoubleNumber("0x0.3p10"), 0.000001);
380382
ASSERT_EQUALS_DOUBLE(5.42101e-20, MathLib::toDoubleNumber("0x1p-64"), 1e-20);

‎test/testsimplifytokens.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1804,7 +1804,7 @@ class TestSimplifyTokens : public TestFixture {
18041804
void combine_wstrings() {
18051805
const char code[] = "a = L\"hello \" L\"world\" ;\n";
18061806

1807-
const char expected[] = "a = \"hello world\" ;";
1807+
const char expected[] = "a = L\"hello world\" ;";
18081808

18091809
Tokenizer tokenizer(&settings0, this);
18101810
std::istringstream istr(code);
@@ -1817,33 +1817,33 @@ class TestSimplifyTokens : public TestFixture {
18171817
void combine_ustrings() {
18181818
const char code[] = "abcd = u\"ab\" u\"cd\";";
18191819

1820-
const char expected[] = "abcd = \"abcd\" ;";
1820+
const char expected[] = "abcd = u\"abcd\" ;";
18211821

18221822
Tokenizer tokenizer(&settings0, this);
18231823
std::istringstream istr(code);
18241824
tokenizer.tokenize(istr, "test.cpp");
18251825

18261826
ASSERT_EQUALS(expected, tokenizer.tokens()->stringifyList(nullptr, false));
1827-
ASSERT_EQUALS(true, tokenizer.tokens()->tokAt(2)->isLong());
1827+
ASSERT_EQUALS(false, tokenizer.tokens()->tokAt(2)->isLong());
18281828
}
18291829

18301830
void combine_Ustrings() {
18311831
const char code[] = "abcd = U\"ab\" U\"cd\";";
18321832

1833-
const char expected[] = "abcd = \"abcd\" ;";
1833+
const char expected[] = "abcd = U\"abcd\" ;";
18341834

18351835
Tokenizer tokenizer(&settings0, this);
18361836
std::istringstream istr(code);
18371837
tokenizer.tokenize(istr, "test.cpp");
18381838

18391839
ASSERT_EQUALS(expected, tokenizer.tokens()->stringifyList(nullptr, false));
1840-
ASSERT_EQUALS(true, tokenizer.tokens()->tokAt(2)->isLong());
1840+
ASSERT_EQUALS(false, tokenizer.tokens()->tokAt(2)->isLong());
18411841
}
18421842

18431843
void combine_u8strings() {
18441844
const char code[] = "abcd = u8\"ab\" u8\"cd\";";
18451845

1846-
const char expected[] = "abcd = \"abcd\" ;";
1846+
const char expected[] = "abcd = u8\"abcd\" ;";
18471847

18481848

18491849
Tokenizer tokenizer(&settings0, this);

‎test/teststring.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,7 @@ class TestString : public TestFixture {
603603
check("int f() {\n"
604604
" return test.substr( 0 , 4 ) == L\"Hello\" ? 0 : 1 ;\n"
605605
"}");
606-
ASSERT_EQUALS("[test.cpp:2]: (warning) String literal \"Hello\" doesn't match length argument for substr().\n", errout.str());
606+
ASSERT_EQUALS("[test.cpp:2]: (warning) String literal L\"Hello\" doesn't match length argument for substr().\n", errout.str());
607607

608608
check("int f() {\n"
609609
" return test.substr( 0 , 5 ) == \"Hello\" ? 0 : 1 ;\n"
@@ -688,7 +688,7 @@ class TestString : public TestFixture {
688688
" int x = 'd' ? 1 : 2;\n"
689689
"}");
690690
ASSERT_EQUALS("[test.cpp:2]: (warning) Conversion of char literal 'a' to bool always evaluates to true.\n"
691-
"[test.cpp:3]: (warning) Conversion of char literal 'b' to bool always evaluates to true.\n"
691+
"[test.cpp:3]: (warning) Conversion of char literal L'b' to bool always evaluates to true.\n"
692692
"[test.cpp:4]: (warning) Conversion of char literal 'c' to bool always evaluates to true.\n"
693693
"[test.cpp:5]: (warning) Conversion of char literal 'd' to bool always evaluates to true.\n"
694694
, errout.str());
@@ -704,7 +704,7 @@ class TestString : public TestFixture {
704704
" if(L'\\0' || cond){}\n"
705705
"}");
706706
ASSERT_EQUALS("[test.cpp:2]: (warning) Conversion of char literal '\\0' to bool always evaluates to false.\n"
707-
"[test.cpp:3]: (warning) Conversion of char literal '\\0' to bool always evaluates to false.\n", errout.str());
707+
"[test.cpp:3]: (warning) Conversion of char literal L'\\0' to bool always evaluates to false.\n", errout.str());
708708
}
709709

710710
void deadStrcmp() {

‎test/testtoken.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -887,7 +887,7 @@ class TestToken : public TestFixture {
887887
tok.concatStr("123");
888888

889889
ASSERT_EQUALS(false, tok.isBoolean());
890-
ASSERT_EQUALS("tru23", tok.str());
890+
ASSERT_EQUALS("tru\"", tok.str());
891891
}
892892

893893
void isNameGuarantees1() const {
@@ -990,6 +990,9 @@ class TestToken : public TestFixture {
990990

991991
givenACodeSampleToTokenize data4("return L\"a\";");
992992
ASSERT_EQUALS("returnL\"a\"", data4.tokens()->expressionString());
993+
994+
givenACodeSampleToTokenize data5("return U\"a\";");
995+
ASSERT_EQUALS("returnU\"a\"", data5.tokens()->expressionString());
993996
}
994997

995998
void hasKnownIntValue() {

‎test/testtokenize.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7222,10 +7222,10 @@ class TestTokenizer : public TestFixture {
72227222
ASSERT_EQUALS("a\"\"=", testAst("a=\"\""));
72237223
ASSERT_EQUALS("a\'\'=", testAst("a=\'\'"));
72247224
ASSERT_EQUALS("'X''a'>", testAst("('X' > 'a')"));
7225-
ASSERT_EQUALS("'X''a'>", testAst("(L'X' > L'a')"));
7226-
ASSERT_EQUALS("'X''a'>", testAst("(u'X' > u'a')"));
7227-
ASSERT_EQUALS("'X''a'>", testAst("(U'X' > U'a')"));
7228-
ASSERT_EQUALS("'X''a'>", testAst("(u8'X' > u8'a')"));
7225+
ASSERT_EQUALS("L'X'L'a'>", testAst("(L'X' > L'a')"));
7226+
ASSERT_EQUALS("u'X'u'a'>", testAst("(u'X' > u'a')"));
7227+
ASSERT_EQUALS("U'X'U'a'>", testAst("(U'X' > U'a')"));
7228+
ASSERT_EQUALS("u8'X'u8'a'>", testAst("(u8'X' > u8'a')"));
72297229

72307230
ASSERT_EQUALS("a0>bc/d:?", testAst("(a>0) ? (b/(c)) : d;"));
72317231
ASSERT_EQUALS("abc/+d+", testAst("a + (b/(c)) + d;"));

0 commit comments

Comments
 (0)