Skip to content
Snippets Groups Projects
Commit 97014b3a authored by g0dil's avatar g0dil
Browse files

Utils/Console: Add position information to Tokens

parent 49cfb353
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Utils/Console/Parse.cc
+ 47
− 7
View file @ 97014b3a
......@@ -112,6 +112,12 @@ namespace detail {
///////////////////////////////////////////////////////////////////////////
// senf::console::Token
prefix_ senf::console::Token::Token(TokenType type, std::string token,
detail::FilePositionWithIndex const & pos)
: type_(type), token_ (token), line_ (pos.line), column_ (pos.column), index_ (pos.index)
{}
prefix_ std::ostream & senf::console::operator<<(std::ostream & os, Token const & token)
{
static char const * tokenTypeName[] = {
......@@ -253,12 +259,43 @@ namespace {
"path expected",
"')' expected",
"'\"' expected" };
boost::spirit::file_position pos (err.where.get_position());
senf::console::detail::FilePositionWithIndex pos (err.where.get_position());
throw senf::console::CommandParser::ParserErrorException(msg[err.descriptor])
<< "\nat " << pos.file << ":" << pos.line << ":" << pos.column;
}
}
namespace boost {
namespace spirit {
template <>
struct position_policy<senf::console::detail::FilePositionWithIndex>
: public position_policy<file_position>
{
typedef position_policy<file_position> Base;
void next_line(senf::console::detail::FilePositionWithIndex & pos)
{
Base::next_line(pos);
pos.index ++;
}
void next_char(senf::console::detail::FilePositionWithIndex & pos)
{
Base::next_char(pos);
pos.index ++;
}
void tabulation(senf::console::detail::FilePositionWithIndex & pos)
{
Base::tabulation(pos);
pos.index ++;
}
};
}}
prefix_ senf::console::CommandParser::CommandParser()
: impl_ (new Impl())
{}
......@@ -273,7 +310,8 @@ template <class Iterator>
prefix_ Iterator senf::console::CommandParser::parseLoop(Iterator npb, Iterator npe,
std::string const & source, Callback cb)
{
typedef boost::spirit::position_iterator<Iterator> PositionIterator;
typedef boost::spirit::position_iterator<
Iterator, detail::FilePositionWithIndex> PositionIterator;
PositionIterator b (npb, npe, source);
PositionIterator e (npe, npe, source);
ParseCommandInfo info;
......@@ -305,7 +343,7 @@ prefix_ Iterator senf::console::CommandParser::parseLoop(Iterator npb, Iterator
cb(info);
}
catch (senf::ExceptionMixin & ex) {
boost::spirit::file_position pos (result.stop.get_position());
detail::FilePositionWithIndex pos (result.stop.get_position());
ex << "\nat " << pos.file << ":" << pos.line << ":" << pos.column;
throw;
}
......@@ -329,7 +367,8 @@ prefix_ void senf::console::CommandParser::parseFile(std::string const & filenam
prefix_ void senf::console::CommandParser::parseArguments(std::string const & arguments,
ParseCommandInfo & info)
{
typedef boost::spirit::position_iterator<std::string::const_iterator> PositionIterator;
typedef boost::spirit::position_iterator<
std::string::const_iterator, detail::FilePositionWithIndex> PositionIterator;
PositionIterator b (arguments.begin(), arguments.end(), std::string("<unknown>"));
PositionIterator e (arguments.end(), arguments.end(), std::string("<unknown>"));
detail::ParseDispatcher::BindInfo bind (impl().dispatcher, info);
......@@ -343,7 +382,7 @@ prefix_ void senf::console::CommandParser::parseArguments(std::string const & ar
throwParserError(ex);
}
if (! result.full) {
boost::spirit::file_position pos (result.stop.get_position());
detail::FilePositionWithIndex pos (result.stop.get_position());
throw ParserErrorException("argument expected")
<< "\nat " << pos.file << ":" << pos.line << ":" << pos.column;
}
......@@ -352,7 +391,8 @@ prefix_ void senf::console::CommandParser::parseArguments(std::string const & ar
prefix_ void senf::console::CommandParser::parsePath(std::string const & path,
ParseCommandInfo & info)
{
typedef boost::spirit::position_iterator<std::string::const_iterator> PositionIterator;
typedef boost::spirit::position_iterator<
std::string::const_iterator, detail::FilePositionWithIndex> PositionIterator;
PositionIterator b (path.begin(), path.end(), std::string("<unknown>"));
PositionIterator e (path.end(), path.end(), std::string("<unknown>"));
detail::ParseDispatcher::BindInfo bind (impl().dispatcher, info);
......@@ -366,7 +406,7 @@ prefix_ void senf::console::CommandParser::parsePath(std::string const & path,
throwParserError(ex);
}
if (! result.full) {
boost::spirit::file_position pos (result.stop.get_position());
detail::FilePositionWithIndex pos (result.stop.get_position());
throw ParserErrorException("path expected")
<< "\nat " << pos.file << ":" << pos.line << ":" << pos.column;
}
......
Utils/Console/Parse.cci
+ 20
− 2
View file @ 97014b3a
......@@ -47,6 +47,24 @@ prefix_ senf::console::Token::TokenType senf::console::Token::type()
return type_;
}
prefix_ unsigned senf::console::Token::line()
const
{
return line_;
}
prefix_ unsigned senf::console::Token::column()
const
{
return column_;
}
prefix_ unsigned senf::console::Token::index()
const
{
return index_;
}
prefix_ bool senf::console::Token::is(unsigned tokens)
const
{
......@@ -66,11 +84,11 @@ prefix_ bool senf::console::Token::operator!=(Token const & other)
}
prefix_ senf::console::Token::Token()
: type_(None), token_()
: type_ (None), token_ ()
{}
prefix_ senf::console::Token::Token(TokenType type, std::string token)
: type_(type), token_ (token)
: type_ (type), token_ (token), line_ (0), column_ (0), index_ (0)
{}
prefix_ senf::console::Token senf::console::NoneToken()
......
Utils/Console/Parse.hh
+ 13
− 1
View file @ 97014b3a
......@@ -205,6 +205,8 @@
namespace senf {
namespace console {
namespace detail { class FilePositionWithIndex; }
namespace detail { struct ParserAccess; }
/** \brief Single argument token
......@@ -253,7 +255,10 @@ namespace console {
};
Token(); ///< Create empty token
Token(TokenType type, std::string token); ///< Create token with given type and value
Token(TokenType type, std::string token);
///< Create token with given type and value
Token(TokenType type, std::string token, detail::FilePositionWithIndex const & pos);
///< Create token with given type and value
std::string const & value() const; ///< String value of token
......@@ -261,6 +266,10 @@ namespace console {
TokenType type() const; ///< Token type
unsigned line() const; ///< Line number of token in source
unsigned column() const; ///< Column number of token in source
unsigned index() const; ///< Index (char count) of token in source
bool is(unsigned tokens) const; ///< Check, whether tokens type matches \a tokens
/**< \a tokens is a bit-mask of token types to check. */
......@@ -272,6 +281,9 @@ namespace console {
private:
TokenType type_;
std::string token_;
unsigned line_;
unsigned column_;
unsigned index_;
};
std::ostream & operator<<(std::ostream & os, Token const & token);
......
Utils/Console/Parse.ih
+ 67
− 29
View file @ 97014b3a
......@@ -43,6 +43,30 @@ namespace detail {
#ifndef DOXYGEN
struct FilePositionWithIndex
: public boost::spirit::file_position
{
int index;
FilePositionWithIndex(std::string const & file_ = std::string(),
int line_ = 1, int column_ = 1, int index_ = 0)
: boost::spirit::file_position (file_, line_, column_), index (index_)
{}
bool operator==(const FilePositionWithIndex & fp) const
{
return boost::spirit::file_position::operator==(fp) && index == fp.index;
}
};
struct PositionOf {
template <class A1> struct result { typedef FilePositionWithIndex type; };
template <class A1> FilePositionWithIndex operator()(A1 & a1) { return a1.get_position(); }
FilePositionWithIndex operator()(char const * a1) { return FilePositionWithIndex(); }
};
::phoenix::function<PositionOf> const positionOf;
///////////////////////////////////////////////////////////////////////////
// Grammar
......@@ -64,6 +88,7 @@ namespace detail {
std::vector<Token> path;
char ch;
Token token;
FilePositionWithIndex pos;
};
Context & context;
......@@ -122,11 +147,12 @@ namespace detail {
using namespace senf::phoenix;
typedef ParseDispatcher PD;
actor< variable< char > > ch_ (self.context.ch);
actor< variable< std::string > > str_ (self.context.str);
actor< variable< std::vector<Token> > > path_ (self.context.path);
actor< variable< Token > > token_ (self.context.token);
actor< variable< ParseDispatcher > > d_ (self.dispatcher);
actor< variable< char > > ch_ (self.context.ch);
actor< variable< std::string > > str_ (self.context.str);
actor< variable< std::vector<Token> > > path_ (self.context.path);
actor< variable< Token > > token_ (self.context.token);
actor< variable< FilePositionWithIndex > > pos_ (self.context.pos);
actor< variable< ParseDispatcher > > d_ (self.dispatcher);
assertion<Errors> end_of_statement_expected (EndOfStatementExpected);
assertion<Errors> path_expected (PathExpected);
......@@ -224,7 +250,7 @@ namespace detail {
;
string // Returns value in context.token
= eps_p [ clear(str_) ]
= eps_p [ pos_ = positionOf(arg1) ][ clear(str_) ]
>> lexeme_d
[
ch_p('"')
......@@ -234,17 +260,19 @@ namespace detail {
)
>> quote_expected(ch_p('"'))
[ token_ = construct_<Token>(Token::BasicString,
str_) ]
str_,
pos_) ]
]
;
hexstring // Returns value in context.token
= eps_p [ clear(str_) ]
= eps_p [ pos_ = positionOf(arg1) ][ clear(str_) ]
>> "x\""
>> * ( hexbyte - ch_p('"') )
>> quote_expected(ch_p('"'))
[ token_ = construct_<Token>(Token::HexString,
str_) ]
str_,
pos_) ]
;
opt_path
......@@ -270,15 +298,19 @@ namespace detail {
;
balanced_tokens
= ch_p('(') [ token_ = construct_<Token>(
= eps_p [ pos_ = positionOf(arg1) ]
>> ch_p('(') [ token_ = construct_<Token>(
Token::ArgumentGroupOpen,
"(") ]
"(",
pos_) ]
[ bind(&PD::pushToken)(d_, token_) ]
>> * token
>> eps_p [ pos_ = positionOf(arg1) ]
>> closing_paren_expected(ch_p(')'))
[ token_ = construct_<Token>(
Token::ArgumentGroupClose,
")") ]
")",
pos_) ]
[ bind(&PD::pushToken)(d_, token_) ]
;
......@@ -289,31 +321,37 @@ namespace detail {
;
punctuation // Returns value in context.str
= ch_p('/') [ token_ = construct_<Token>(
Token::PathSeparator,
"/") ]
| ch_p('{') [ token_ = construct_<Token>(
Token::DirectoryGroupOpen,
"{") ]
| ch_p('}') [ token_ = construct_<Token>(
Token::DirectoryGroupClose,
"}") ]
| ch_p(';') [ token_ = construct_<Token>(
Token::CommandTerminator,
";") ]
| self.punctuation_p [ token_ = construct_<Token>(
Token::OtherPunctuation,
construct_<std::string>(1u, arg1)) ]
= eps_p [ pos_ = positionOf(arg1) ]
>> (
ch_p('/') [ token_ = construct_<Token>(
Token::PathSeparator,
"/") ]
| ch_p('{') [ token_ = construct_<Token>(
Token::DirectoryGroupOpen,
"{") ]
| ch_p('}') [ token_ = construct_<Token>(
Token::DirectoryGroupClose,
"}") ]
| ch_p(';') [ token_ = construct_<Token>(
Token::CommandTerminator,
";") ]
| self.punctuation_p [ token_ = construct_<Token>(
Token::OtherPunctuation,
construct_<std::string>(1u, arg1),
pos_) ]
)
;
word // Returns value in context.token
= lexeme_d
= eps_p [ pos_ = positionOf(arg1) ]
>> lexeme_d
[
(+ self.word_p) [ str_ = construct_<std::string>(arg1, arg2) ]
]
>> eps_p [ token_ = construct_<Token>(
Token::Word,
str_) ]
str_,
pos_) ]
;
hexbyte
......
Utils/Console/Parse.test.cc
+ 7
− 2
View file @ 97014b3a
......@@ -85,7 +85,7 @@ BOOST_AUTO_UNIT_TEST(commandGrammar)
{
static char text[] =
"# Comment\n"
"doo/bii/doo arg/two/three"
"doo / bii / doo arg"
" flab::blub"
" 123.434>a"
" (a,b;c (huhu/{haha}))"
......@@ -99,7 +99,7 @@ BOOST_AUTO_UNIT_TEST(commandGrammar)
grammar.use_parser<Grammar::SkipParser>() ) . full );
BOOST_CHECK_EQUAL( ss.str(),
"beginCommand( Word('doo')/Word('bii')/Word('doo') )\n"
"pushToken( Word('arg/two/three') )\n"
"pushToken( Word('arg') )\n"
"pushToken( Word('flab::blub') )\n"
"pushToken( Word('123.434>a') )\n"
"pushToken( ArgumentGroupOpen('(') )\n"
......@@ -214,6 +214,7 @@ BOOST_AUTO_UNIT_TEST(commandParser)
BOOST_CHECK_EQUAL_COLLECTIONS( info.commandPath().begin(), info.commandPath().end(),
path, path + sizeof(path)/sizeof(path[0]) );
BOOST_CHECK_EQUAL( boost::next(info.commandPath().begin())->index(), 16u );
BOOST_CHECK_EQUAL( unsigned(info.tokens().size()), 15u );
char const * tokens[] = { "arg",
......@@ -243,11 +244,15 @@ BOOST_AUTO_UNIT_TEST(commandParser)
BOOST_REQUIRE_EQUAL( args->size(), 8u );
for (unsigned i (0); i<8; ++i)
BOOST_CHECK_EQUAL( args->begin()[i].value(), tokens[4+i] );
BOOST_CHECK_EQUAL( info.tokens().begin()[3].index(), 96u );
BOOST_CHECK_EQUAL( info.tokens().begin()[5].index(), 98u );
BOOST_CHECK_EQUAL( info.tokens().begin()[12].index(), 109u );
++ args;
BOOST_REQUIRE( args != info.arguments().end() );
BOOST_REQUIRE_EQUAL( args->size(), 1u );
BOOST_CHECK_EQUAL( args->begin()->value(), tokens[13] );
BOOST_CHECK_EQUAL( args->begin()->index(), 126u );
++ args;
BOOST_REQUIRE( args != info.arguments().end() );
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Datenschutz und Impressum