Boost Spirit grammar eol

Question

I am trying to parse files of the following form:

// comment bla bla
[sectionname]
key = value
key2=value2


// comment
key = value

[anothersection]
...

using the following code. Unfortunately, it reports the last eol as an error although all eols at the end should be accepted by: (*qi::eol > -(sectionGrammar > *(+qi::eol > sectionGrammar)) > *qi::eol),

Besides I really don't know how to parse comments properly without taking the eol which is required for the next key-value pair which is the reason I didn't placed in in the Skipper (only ascii::blank).

The last issue I have is that I don't know how to add sections to a boost::ptr_vector without copying them.

This is my code:

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/support_multi_pass.hpp>
#include <boost/spirit/include/classic_position_iterator.hpp> // for more detailed error information

#include <boost/fusion/adapted/std_pair.hpp>
#include <boost/fusion/adapted/struct/adapt_struct.hpp>
#include <boost/fusion/include/adapt_struct.hpp>

#include <boost/bind.hpp>
#include <boost/spirit/home/phoenix/core/argument.hpp>

#include <boost/foreach.hpp>

#include "txt.hpp"

// Only use in global namespace!
BOOST_FUSION_ADAPT_STRUCT(
    wc3lib::map::Txt::Section,
    (wc3lib::string, name)
    (wc3lib::map::Txt::Pairs, entries)
)

namespace wc3lib
{

namespace map
{

namespace client
{

    using namespace boost::spirit;
//using namespace boost::spirit::qi;
using qi::double_;
using qi::phrase_parse;
using standard::space;
using boost::phoenix::ref;

//typedef BOOST_TYPEOF(space | lit("//") >> *(standard::char_ - qi::eol) >> qi::eol) SkipperType;

/*
 * Doesn't skip eols since value pairs are separated linewise which therefore can be specified easier in the rules
 */
template<typename Iterator>
struct CommentSkipper : public qi::grammar<Iterator> {

    qi::rule<Iterator> skip;

    CommentSkipper() : CommentSkipper::base_type(skip, "PL/0")
    {
        skip = ascii::blank | lit("//") >> *(standard::char_ - qi::eol) >> qi::eol;
    }
};

template <typename Iterator, typename Skipper = CommentSkipper<Iterator> >
struct KeyValueSquence : qi::grammar<Iterator, Txt::Pairs(), Skipper>
{
    //Txt::Pairs::value_type
    qi::rule<Iterator, Txt::Pairs(), Skipper> query; // NOTE first rule used as parameter for base_type does always need the skipper type of the grammar
    qi::rule<Iterator, std::pair<string, string>(), Skipper> pair;
    qi::rule<Iterator, string()> key, value;

    KeyValueSquence() : KeyValueSquence::base_type(query)
    {
        query =  pair > *(pair); // use only > for backtracking
        pair  =  +qi::eol > key > lit('=') > -value; // -('=' >> value)
        key   =  standard::char_("a-zA-Z_") > *standard::char_("a-zA-Z_0-9");
        value = +(standard::char_ - qi::eol); // values can be empty or all characters except eol which indicates the and of the value
    }
};

template <typename Iterator, typename Skipper = CommentSkipper<Iterator> >
struct SectionRule : qi::grammar<Iterator, Txt::Section(), Skipper>
{
    qi::rule<Iterator, Txt::Section(), Skipper> query;
    qi::rule<Iterator, string()> name;
    qi::rule<Iterator, Txt::Pairs(), Skipper> entries;

    KeyValueSquence<Iterator, Skipper> keyValueSequence;

    SectionRule() : SectionRule::base_type(query)
    {
        query =  name > -entries;
        name  =  lit('[') > standard::char_("a-zA-Z_") > *standard::char_("a-zA-Z_0-9") > lit(']');
        entries = keyValueSequence;
    }
};

template <typename Iterator>
bool parse(Iterator first, Iterator last, Txt::Sections &sections)
{
    SectionRule<Iterator> sectionGrammar;
    CommentSkipper<Iterator> commentSkipper;
    std::vector<Txt::Section> tmpSections;

    bool r = boost::spirit::qi::phrase_parse(
    first,
    last,
    (*qi::eol > -(sectionGrammar > *(+qi::eol > sectionGrammar)) > *qi::eol),
    // comment skipper
    commentSkipper,
    tmpSections //sections store into "sections"!
    );

    if (first != last) // fail if we did not get a full match
    {
        return false;
    }

    // TODO temporary workaround, add sections directly from heap to vector
    BOOST_FOREACH(std::vector<Txt::Section>::const_reference ref, tmpSections) {
        std::auto_ptr<Txt::Section> s(new Txt::Section());
        s->name = ref.name;
        s->entries = ref.entries;
        sections.push_back(s);
    }

    return r;
}

}
</code>

Answer 1

From the comment

// use only > for backtracking

I get the impression that you're understanding this wrong. > will actually prevent backtracking past that point, because it mandates the next token.

In order to present some techniques, I've mocked up the presumed missing header:

// #include "txt.hpp"
// minimal mockup
namespace wc3lib { 
    using std::string;

    namespace map { namespace Txt { 
            typedef std::map<string, string> Pairs;

            struct Section
            {
                string name;
                Pairs entries;
            };

            typedef std::vector<Section> Sections;
    } }
}

Now, I've "fixed" up your code to show how to do

• debugging
• error handling
• expectations (I've opted for the 'non-strict' solution, because I don't have time to watch closely for which expectations make sense)

See it live at Coliru

Note

• please don't using namespace at namespace scope. Instead, use convenient namespace aliases:

  namespace qi = boost::spirit::qi;
  namespace ascii = boost::spirit::ascii;
  

• please don't (don't) use auto_ptr<>. It's error-prone, obsolete, deprecated, inflexible :/ Use unique_ptr instead if you must have the sections on the heap (why?).

• expect the end of the stream as well as eol to terminate your lines...

• be careful of naming.

  • query was double used as a rule name
  • are you parsing sections or queries or KeyValueSquences?
  • consider using section_header instead of name etc.

  I do have the feeling some confusion would not have occurred with more careful naming.

• consider merging the grammars into one, unless

  • your real grammar is way more complicated (at this level, that seems to be irrelevant, since you could still group these three grammars into one subgrammar)
  • you absolutely need to split the grammars across translation units

Without further ado:

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>

#include <boost/fusion/adapted.hpp>
#include <boost/foreach.hpp>
#include <map>

// #include "txt.hpp"
// minimal mockup
namespace wc3lib { 
    using std::string;

    namespace map { namespace Txt { 
            typedef std::map<string, string> Pairs;

            struct Section
            {
                string name;
                Pairs entries;
            };

            typedef std::vector<Section> Sections;
    } }
}

// Only use in global namespace!
BOOST_FUSION_ADAPT_STRUCT(
    wc3lib::map::Txt::Section,
    (wc3lib::string, name)
    (wc3lib::map::Txt::Pairs, entries)
)

namespace wc3lib { namespace map { namespace client {

    namespace qi = boost::spirit::qi;
    namespace ascii = boost::spirit::ascii;

    /*
     * Doesn't skip eols since value pairs are separated linewise which therefore can be specified easier in the rules
     */
    template<typename Iterator>
    struct CommentSkipper : public qi::grammar<Iterator> {

        qi::rule<Iterator> skip;

        CommentSkipper() : CommentSkipper::base_type(skip, "PL/0")
        {
            using namespace qi;
            skip = ascii::blank | (lit("//") >> *(standard::char_ - eol) >> eol);

            BOOST_SPIRIT_DEBUG_NODES((skip));
        }
    };

    template <typename Iterator, typename Skipper = CommentSkipper<Iterator> >
    struct KeyValueSquence : qi::grammar<Iterator, Txt::Pairs(), Skipper>
    {
        qi::rule<Iterator, Txt::Pairs(), Skipper> pairs; // NOTE first rule used as parameter for base_type does always need the skipper type of the grammar
        qi::rule<Iterator, std::pair<string, string>(), Skipper> pair;
        qi::rule<Iterator, string()> key, value;

        KeyValueSquence() : KeyValueSquence::base_type(pairs)
        {
            using namespace qi;

            pairs = +pair; // use only > for backtracking

            // these had a problem with backtracking (failing the rule at the end of a section)
            pair  = +eol > key > lit('=') > value; // -('=' >> value)
            key   = standard::char_("a-zA-Z_") > *standard::char_("a-zA-Z_0-9");

            // using this removes that problem:
            pair  = +eol >> key >> lit('=') >> value; // -('=' >> value)
            key   = standard::char_("a-zA-Z_") >> *standard::char_("a-zA-Z_0-9");

            value = *(standard::char_ - (eol|eoi)); // values can be empty or all characters except eol which indicates the end of the value

            BOOST_SPIRIT_DEBUG_NODES((pairs)(pair)(key)(value));
        }
    };

    template <typename Iterator, typename Skipper = CommentSkipper<Iterator> >
    struct SectionRule : qi::grammar<Iterator, Txt::Section(), Skipper>
    {
        qi::rule<Iterator, Txt::Section(), Skipper> query;
        qi::rule<Iterator, string()> name;

        KeyValueSquence<Iterator, Skipper> keyValueSequence;

        SectionRule() : SectionRule::base_type(query)
        {
            using namespace qi;

            name  =  lit('[') > standard::char_("a-zA-Z_") > *standard::char_("a-zA-Z_0-9") > lit(']');
            query =  name > -keyValueSequence;

            BOOST_SPIRIT_DEBUG_NODES((query)(name));
        }
    };

    template <typename Iterator>
    bool parse(Iterator first, Iterator last, Txt::Sections &sections)
    {
        SectionRule<Iterator> sectionGrammar;
        CommentSkipper<Iterator> commentSkipper;
        std::vector<Txt::Section> tmpSections;

        try
        {
            bool r = qi::phrase_parse(
                    first, last,
                    (sectionGrammar % +qi::eol) >> *qi::eol > qi::eoi,
                    // comment skipper
                    commentSkipper,
                    tmpSections //sections store into "sections"!
                    );

            if (first != last) // fail if we did not get a full match
            {
                std::cerr << "DEBUG: Unparsed: '" << std::string(first,last) << "\n";
                return false;
            }
            // TODO temporary workaround, add sections directly from heap to vector
            sections = tmpSections;

            return r;
        } catch(qi::expectation_failure<Iterator> const& e)
        {
            std::cerr << "Unexpected: " << e.what() << " at '" << std::string(e.first,e.last) << "\n";
            return false;
        }
    }

} } }

int main()
{
    std::cin.unsetf(std::ios::skipws);
    boost::spirit::istream_iterator f(std::cin), l;

    wc3lib::map::Txt::Sections parsed;
    bool ok = wc3lib::map::client::parse(f, l, parsed);
    if (ok)
    {
        std::cout << "Parsed " << parsed.size() << " sections\n";
        for(auto& section : parsed)
        {
            std::cout << "section [" << section.name << "] has " << section.entries.size() << " pairs\n";
        }
    }
}

Which prints

Parsed 2 sections
section [sectionname] has 3 pairs
section [anothersection] has 1 pairs

After/along with the debug trace of the parsers:

<query>
  <try>// comment bla bla\n[</try>
  <skip>
    <try>// comment bla bla\n[</try>
    <success>[sectionname]\nkey = </success>
    <attributes>[]</attributes>
  </skip>
  <skip>
    <try>[sectionname]\nkey = </try>
    <fail/>
  </skip>
  <name>
    <try>[sectionname]\nkey = </try>
    <success>\nkey = value\nkey2=va</success>
    <attributes>[[s, e, c, t, i, o, n, n, a, m, e]]</attributes>
  </name>
  <pairs>
    <try>\nkey = value\nkey2=va</try>
    <pair>
      <try>\nkey = value\nkey2=va</try>
      <skip>
        <try>\nkey = value\nkey2=va</try>
        <fail/>
      </skip>
      <skip>
        <try>key = value\nkey2=val</try>
        <fail/>
      </skip>
      <skip>
        <try>key = value\nkey2=val</try>
        <fail/>
      </skip>
      <key>
        <try>key = value\nkey2=val</try>
        <success> = value\nkey2=value2</success>
        <attributes>[[k, e, y]]</attributes>
      </key>
      <skip>
        <try> = value\nkey2=value2</try>
        <success>= value\nkey2=value2\n</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>= value\nkey2=value2\n</try>
        <fail/>
      </skip>
      <skip>
        <try> value\nkey2=value2\n\n</try>
        <success>value\nkey2=value2\n\n\n</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>value\nkey2=value2\n\n\n</try>
        <fail/>
      </skip>
      <value>
        <try>value\nkey2=value2\n\n\n</try>
        <success>\nkey2=value2\n\n\n// co</success>
        <attributes>[[v, a, l, u, e]]</attributes>
      </value>
      <success>\nkey2=value2\n\n\n// co</success>
      <attributes>[[[k, e, y], [v, a, l, u, e]]]</attributes>
    </pair>
    <pair>
      <try>\nkey2=value2\n\n\n// co</try>
      <skip>
        <try>\nkey2=value2\n\n\n// co</try>
        <fail/>
      </skip>
      <skip>
        <try>key2=value2\n\n\n// com</try>
        <fail/>
      </skip>
      <skip>
        <try>key2=value2\n\n\n// com</try>
        <fail/>
      </skip>
      <key>
        <try>key2=value2\n\n\n// com</try>
        <success>=value2\n\n\n// comment</success>
        <attributes>[[k, e, y, 2]]</attributes>
      </key>
      <skip>
        <try>=value2\n\n\n// comment</try>
        <fail/>
      </skip>
      <skip>
        <try>value2\n\n\n// comment\n</try>
        <fail/>
      </skip>
      <value>
        <try>value2\n\n\n// comment\n</try>
        <success>\n\n\n// comment\nkey3 =</success>
        <attributes>[[v, a, l, u, e, 2]]</attributes>
      </value>
      <success>\n\n\n// comment\nkey3 =</success>
      <attributes>[[[k, e, y, 2], [v, a, l, u, e, 2]]]</attributes>
    </pair>
    <pair>
      <try>\n\n\n// comment\nkey3 =</try>
      <skip>
        <try>\n\n\n// comment\nkey3 =</try>
        <fail/>
      </skip>
      <skip>
        <try>\n\n// comment\nkey3 = </try>
        <fail/>
      </skip>
      <skip>
        <try>\n// comment\nkey3 = v</try>
        <fail/>
      </skip>
      <skip>
        <try>// comment\nkey3 = va</try>
        <success>key3 = value3\n\n[anot</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>key3 = value3\n\n[anot</try>
        <fail/>
      </skip>
      <skip>
        <try>key3 = value3\n\n[anot</try>
        <fail/>
      </skip>
      <key>
        <try>key3 = value3\n\n[anot</try>
        <success> = value3\n\n[anothers</success>
        <attributes>[[k, e, y, 3]]</attributes>
      </key>
      <skip>
        <try> = value3\n\n[anothers</try>
        <success>= value3\n\n[anotherse</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>= value3\n\n[anotherse</try>
        <fail/>
      </skip>
      <skip>
        <try> value3\n\n[anothersec</try>
        <success>value3\n\n[anothersect</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>value3\n\n[anothersect</try>
        <fail/>
      </skip>
      <value>
        <try>value3\n\n[anothersect</try>
        <success>\n\n[anothersection]\nk</success>
        <attributes>[[v, a, l, u, e, 3]]</attributes>
      </value>
      <success>\n\n[anothersection]\nk</success>
      <attributes>[[[k, e, y, 3], [v, a, l, u, e, 3]]]</attributes>
    </pair>
    <pair>
      <try>\n\n[anothersection]\nk</try>
      <skip>
        <try>\n\n[anothersection]\nk</try>
        <fail/>
      </skip>
      <skip>
        <try>\n[anothersection]\nke</try>
        <fail/>
      </skip>
      <skip>
        <try>[anothersection]\nkey</try>
        <fail/>
      </skip>
      <skip>
        <try>[anothersection]\nkey</try>
        <fail/>
      </skip>
      <key>
        <try>[anothersection]\nkey</try>
        <fail/>
      </key>
      <fail/>
    </pair>
    <success>\n\n[anothersection]\nk</success>
    <attributes>[[[[k, e, y], [v, a, l, u, e]], [[k, e, y, 2], [v, a, l, u, e, 2]], [[k, e, y, 3], [v, a, l, u, e, 3]]]]</attributes>
  </pairs>
  <success>\n\n[anothersection]\nk</success>
  <attributes>[[[s, e, c, t, i, o, n, n, a, m, e], [[[k, e, y], [v, a, l, u, e]], [[k, e, y, 2], [v, a, l, u, e, 2]], [[k, e, y, 3], [v, a, l, u, e, 3]]]]]</attributes>
</query>
<skip>
  <try>\n\n[anothersection]\nk</try>
  <fail/>
</skip>
<skip>
  <try>\n[anothersection]\nke</try>
  <fail/>
</skip>
<skip>
  <try>[anothersection]\nkey</try>
  <fail/>
</skip>
<query>
  <try>[anothersection]\nkey</try>
  <skip>
    <try>[anothersection]\nkey</try>
    <fail/>
  </skip>
  <name>
    <try>[anothersection]\nkey</try>
    <success>\nkey = value\n</success>
    <attributes>[[a, n, o, t, h, e, r, s, e, c, t, i, o, n]]</attributes>
  </name>
  <pairs>
    <try>\nkey = value\n</try>
    <pair>
      <try>\nkey = value\n</try>
      <skip>
        <try>\nkey = value\n</try>
        <fail/>
      </skip>
      <skip>
        <try>key = value\n</try>
        <fail/>
      </skip>
      <skip>
        <try>key = value\n</try>
        <fail/>
      </skip>
      <key>
        <try>key = value\n</try>
        <success> = value\n</success>
        <attributes>[[k, e, y]]</attributes>
      </key>
      <skip>
        <try> = value\n</try>
        <success>= value\n</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>= value\n</try>
        <fail/>
      </skip>
      <skip>
        <try> value\n</try>
        <success>value\n</success>
        <attributes>[]</attributes>
      </skip>
      <skip>
        <try>value\n</try>
        <fail/>
      </skip>
      <value>
        <try>value\n</try>
        <success>\n</success>
        <attributes>[[v, a, l, u, e]]</attributes>
      </value>
      <success>\n</success>
      <attributes>[[[k, e, y], [v, a, l, u, e]]]</attributes>
    </pair>
    <pair>
      <try>\n</try>
      <skip>
        <try>\n</try>
        <fail/>
      </skip>
      <key>
        <try></try>
        <fail/>
      </key>
      <fail/>
    </pair>
    <success>\n</success>
    <attributes>[[[[k, e, y], [v, a, l, u, e]]]]</attributes>
  </pairs>
  <success>\n</success>
  <attributes>[[[a, n, o, t, h, e, r, s, e, c, t, i, o, n], [[[k, e, y], [v, a, l, u, e]]]]]</attributes>
</query>
<skip>
  <try>\n</try>
  <fail/>
</skip>
<query>
  <try></try>
  <name>
    <try></try>
    <fail/>
  </name>
  <fail/>
</query>
<skip>
  <try>\n</try>
  <fail/>
</skip>