boost spirit: copy the result in a vector of strings

Question

I want to parse a function (with an arbitrary name and an arbitrary numbers af arguments) in this form:

function(bye, 1, 3, 4, foo)

The arguments could be generic strings comma separated. And I want to copy the name of the function and the arguments in a vector of strings. like this

   std::vector<std::string> F;
   std::string fun = "function(bye, 1, 3, 4, foo)";

// The parser must produce this vector from the example
    F[0] == "function"
    F[1] == "1"
    F[2] == "3"
    F[3] == "4"
    F[4] == "foo"

I've written the following code by after reading some tutorial but it does not work (In the sense that it not compile).

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>

#include <iostream>
#include <string>



namespace client
{

    namespace qi = boost::spirit::qi;
    namespace ascii = boost::spirit::ascii;

    ///////////////////////////////////////////////////////////////////////////////
    template <typename Iterator>
    struct command_parser : qi::grammar<Iterator, std::vector<std::string>(), ascii::space_type>
    {
        command_parser() : command_parser::base_type(start)
        {
            using qi::int_;
            using qi::lit;
            using qi::double_;
            using qi::lexeme;
            using ascii::char_;

            fn_name = +qi::char_("a-zA-Z");
            string =  +qi::char_("a-zA-Z_0-9");
            rec = *( lit(",") >> string );

            start %= fn_name >> lit("(") >> string >> rec >> lit(")") ;
        }

        qi::rule<Iterator, std::string(), ascii::space_type> fn_name;
        qi::rule<Iterator, std::string(), ascii::space_type> string;
        qi::rule<Iterator, std::string(), ascii::space_type> rec;

        qi::rule<Iterator, std::vector<std::string>, ascii::space_type> start;
    };
}


////////////////////////////////////////////////////////////////////////////
//  Main program
////////////////////////////////////////////////////////////////////////////
int
 main()
{

    namespace qi = boost::spirit::qi;

    std::cout << "/////////////////////////////////////////////////////////\n\n";

    client::command_parser<std::string::iterator> CP;
    std::string cmd("fun(1,2,3,4  , 5, foo) ");

    std::vector<std::string> VV;

    bool result = qi::parse(cmd.begin(), cmd.end(), CP, VV); 

    if (result) {
        for ( auto sss : VV ){
            std::cout << sss << std::endl;
        }
    } else {
        std::cout << "Fail" << std::endl;
    }

    return 0 ;

}

Answer 1

Just for fun, here's my minimalist take on this grammar:

using CallList = std::vector<std::string>;

struct ParseError : std::runtime_error {
    ParseError() : std::runtime_error("ParseError") {}
};

// The parse implementation
CallList parse_function_call(std::string const& fun) {
    CallList elements;
    using namespace boost::spirit::qi;
    using It = decltype(begin(fun));
    static const rule<It, std::string()> identifier = alpha >> +(alnum | char_('_'));

    if (!phrase_parse(begin(fun), end(fun),
                identifier >> '(' >> -(lexeme[+~char_(",)")] % ",") >> ')' >> eoi,
                space, elements))
        throw ParseError{};
    return elements;
}

With a little bit of plumbing

// just for test output
using TestResult = std::variant<CallList, ParseError>;

// exceptions are equivalent
static constexpr bool operator==(ParseError const&, ParseError const&)
    { return true; }

static inline std::ostream& operator<<(std::ostream& os, TestResult const& tr) {
    using namespace std;
    if (holds_alternative<ParseError>(tr)) {
        return os << "ParseError";
    } else {
        auto& list = get<CallList>(tr);
        copy(begin(list), end(list), std::experimental::make_ostream_joiner(os << "{", ","));
        return os << "}";
    }
}

TestResult try_parse(std::string const& fun) {
    try { return parse_function_call(fun); }
    catch(ParseError const& e) { return e; }
}

Here's a test runner:

for (auto const& [input, expected]: {
        Case("function(bye, 1, 3, 4, foo)", CallList{"function", "1", "3", "4", "foo"}),
        {"liar(pants on fire)", CallList{"liar", "pants on fire"}},
        {"liar('pants on fire')", CallList{"liar", "'pants on fire'"}},
        {"nullary()", CallList{"nullary"}},
        {"nullary(    )", CallList{"nullary"}},
        {"zerolength(a,,b)", ParseError{}},
        {"zerolength(a, ,b)", ParseError{}},
        {"noarglust", ParseError{}},
        {"", ParseError{}},
        {"()", ParseError{}},
        {"1(invalidfunctionname)", ParseError{}},
        {"foo(bar) BOGUS", ParseError{}},
    })
{
    auto const actual = try_parse(input);
    bool const ok = (actual == expected);

    cout << std::quoted(input) << ": " << (ok? "PASS":"FAIL") << "\n";
    if (!ok) {
        std::cout << " -- expected: " << expected << "\n";
        std::cout << " -- actual:   " << actual << "\n";
    }
}

Which prints Live On Coliru

"function(bye, 1, 3, 4, foo)": FAIL
 -- expected: {function,1,3,4,foo}
 -- actual:   {function,bye,1,3,4,foo}
"liar(pants on fire)": PASS
"liar('pants on fire')": PASS
"nullary()": PASS
"nullary(    )": PASS
"zerolength(a,,b)": PASS
"zerolength(a, ,b)": PASS
"noarglust": PASS
"": PASS
"()": PASS
"1(invalidfunctionname)": PASS
"foo(bar) BOGUS": PASS

Note that your example test-case doesn't pass, but I think that was a mistake in the test case.

Full Listing

Live On Coliru

//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <experimental/iterator>
#include <variant>
#include <iomanip>

using CallList = std::vector<std::string>;

struct ParseError : std::runtime_error {
    ParseError() : std::runtime_error("ParseError") {}
};

// The parse implementation
CallList parse_function_call(std::string const& fun) {
    CallList elements;
    using namespace boost::spirit::qi;
    using It = decltype(begin(fun));
    static const rule<It, std::string()> identifier = alpha >> +(alnum | char_('_'));

    if (!phrase_parse(begin(fun), end(fun),
                identifier >> '(' >> -(lexeme[+~char_(",)")] % ",") >> ')' >> eoi,
                space, elements))
        throw ParseError{};
    return elements;
}

// just for test output
using TestResult = std::variant<CallList, ParseError>;

// exceptions are equivalent
static constexpr bool operator==(ParseError const&, ParseError const&)
    { return true; }

static inline std::ostream& operator<<(std::ostream& os, TestResult const& tr) {
    using namespace std;
    if (holds_alternative<ParseError>(tr)) {
        return os << "ParseError";
    } else {
        auto& list = get<CallList>(tr);
        copy(begin(list), end(list), std::experimental::make_ostream_joiner(os << "{", ","));
        return os << "}";
    }
}

TestResult try_parse(std::string const& fun) {
    try { return parse_function_call(fun); }
    catch(ParseError const& e) { return e; }
}

int main() {
    using namespace std;

    using Case = pair<std::string, TestResult>;

    for (auto const& [input, expected]: {
            Case("function(bye, 1, 3, 4, foo)", CallList{"function", "1", "3", "4", "foo"}),
            {"liar(pants on fire)", CallList{"liar", "pants on fire"}},
            {"liar('pants on fire')", CallList{"liar", "'pants on fire'"}},
            {"nullary()", CallList{"nullary"}},
            {"nullary(    )", CallList{"nullary"}},
            {"zerolength(a,,b)", ParseError{}},
            {"zerolength(a, ,b)", ParseError{}},
            {"noarglust", ParseError{}},
            {"", ParseError{}},
            {"()", ParseError{}},
            {"1(invalidfunctionname)", ParseError{}},
            {"foo(bar) BOGUS", ParseError{}},
        })
    {
        auto const actual = try_parse(input);
        bool const ok = (actual == expected);

        cout << std::quoted(input) << ": " << (ok? "PASS":"FAIL") << "\n";
        if (!ok) {
            std::cout << " -- expected: " << expected << "\n";
            std::cout << " -- actual:   " << actual << "\n";
        }
    }
}

Answer 2

First is that for qi::grammer, std::vector() is not a type, it is an object so:

struct command_parser : qi::grammar, ascii::space_type>

Then, you are defining a parser that uses a skipper so qi::parse(...) won't do it, you need to use qi::phrase_parse(...)

start %= won't work as you don't have a container of your_function containers. So you would go with just one function at a time. And I'd quess that is what you want. Or you would use

std::vector<std::vector<std::sting> > >

as your attribute if you want multiple functions.

Now it parses, but not the results you expect. The rec rule parses to a std::string so piles all its stuff into a string before passing it to start make the rule as:

qi::rule<Iterator, std::vector<std::string>, ascii::space_type> rec;

so it is compatible with your attribute.

You should not try to hack from an example without understanding how it works. You will just end up down a rabbit hole. Build from very simple up to what you want a step at a time.

I will admit, this is weird for me as I have not used spirit in years. These days I use spirit::x3. Spirit is painfully slow to compile and a lot more verbose.

Working example for a single function:

namespace client
{
    namespace qi = boost::spirit::qi;
    namespace ascii = boost::spirit::ascii;
    template <typename Iterator>
    struct command_parser : qi::grammar<Iterator, std::vector<std::string>, ascii::space_type>
    {
        command_parser() : command_parser::base_type(start)
        {
            using qi::int_;
            using qi::lit;
            using qi::double_;
            using qi::lexeme;
            using ascii::char_;

            fn_name = +qi::char_("a-zA-Z");
            string = +qi::char_("a-zA-Z_0-9");
            rec = *(lit(",") >> string);
            start = fn_name >> lit("(") >> string >> rec >> lit(")");
        }

        qi::rule<Iterator, std::string(), ascii::space_type> fn_name;
        qi::rule<Iterator, std::string(), ascii::space_type> string;
        qi::rule<Iterator, std::vector<std::string>, ascii::space_type> rec;
        qi::rule<Iterator, std::vector<std::string>, ascii::space_type> start;
    };
}

int main()
{
    namespace qi = boost::spirit::qi;
    client::command_parser<std::string::iterator> CP;
    std::string cmd("fun(1,2,3,4  , 5, foo) ");

    using boost::spirit::ascii::space;
    std::vector<std::string> VV;
    auto it = cmd.begin();
    bool result = qi::phrase_parse(it, cmd.end(), CP, space, VV);

    if (result) {
        for (auto sss : VV) {
            std::cout << sss << std::endl;
        }
    }
    else {
        std::cout << "Fail" << std::endl;
    }
    return 0;
}

ADDED:

So you can see what it looks like using spirit::x3. I think this is much easier on the eye.

namespace client {
    using attr = std::vector<std::string>;
    namespace x3 = boost::spirit::x3;

    const auto fn_name = +x3::char_("a-zA-Z");
    auto string = +x3::char_("a-zA-Z_0-9");
    auto start = x3::rule<struct _, attr>() = fn_name >> "(" >> string % ',' >> ")";
}

int main()
{
    namespace x3 = boost::spirit::x3;
    client::attr VV;
    auto parser = client::start;
    std::string cmd("fun(1,2,3,4  , 5, foo) ");
    auto it = cmd.begin();
    bool result = phrase_parse(it, cmd.end(), parser, x3::space, VV);

    if (result) {
        for (auto sss : VV) {
            std::cout << "-> " << sss << std::endl;
        }
    }
    else {
        std::cout << "Fail" << std::endl;
    }
    return 0;
}