platform/include/pion/platform/Transform.hpp

// ------------------------------------------------------------------------
// Pion is a development platform for building Reactors that process Events
// ------------------------------------------------------------------------
// Copyright (C) 2007-2008 Atomic Labs, Inc.  (http://www.atomiclabs.com)
//
// Pion is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Pion is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pion.  If not, see <http://www.gnu.org/licenses/>.
//

#ifndef __PION_TRANSFORM_HEADER__
#define __PION_TRANSFORM_HEADER__

#include <cctype>
#include <cstring>
#include <set>
#include <boost/regex.hpp>
#include <boost/regex/icu.hpp>
#include <boost/algorithm/string/compare.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/tokenizer.hpp>
#include <pion/PionAlgorithms.hpp>
#include <pion/PionConfig.hpp>
#include <pion/PionException.hpp>
#include <pion/platform/Vocabulary.hpp>
#include <pion/platform/Event.hpp>
#include <pion/platform/Comparison.hpp>
#include <pion/PionLogger.hpp>
#include <pion/platform/ConfigManager.hpp>

namespace pion {        // begin namespace pion
namespace platform {    // begin namespace platform (Pion Platform Library)

class PION_PLATFORM_API Transform
{
public:
    Vocabulary::Term            m_term;

    class MissingTransformField : public PionException {
    public:
        MissingTransformField(const std::string& str)
            : PionException("Invalid type of transformation: ", str) {}
    };

    class RegexFailure : public PionException {
    public:
        RegexFailure(const std::string& what)
            : PionException("Regex replace failed: ", what) {}
    };

    class ValueAssignmentException : public PionException {
    public:
        ValueAssignmentException(const std::string& value)
            : PionException("AssignValue failed for value: ", value) {}
    };

//  mutable PionLogger          m_logger;

    virtual ~Transform() {}

    Transform(const Vocabulary& v, const Vocabulary::Term& term)
        :   m_term(term) //, m_logger(PION_GET_LOGGER("pion.Transform"))
    {
    }

    void updateVocabulary(const Vocabulary& v)
    {
        // assume that Term references never change
        m_term = v[m_term.term_ref];
    }

    inline void removeTerm(EventPtr& e)
    {
        e->clear(m_term.term_ref);
    }

    virtual bool transform(EventPtr& d, const EventPtr& s) = 0;

    static const std::string            LOOKUP_TERM_NAME;
    static const std::string            TERM_ELEMENT_NAME;
    static const std::string            LOOKUP_MATCH_ELEMENT_NAME;
    static const std::string            LOOKUP_FORMAT_ELEMENT_NAME;
    static const std::string            LOOKUP_DEFAULT_ELEMENT_NAME;
    static const std::string            VALUE_ELEMENT_NAME;
    static const std::string            RULES_STOP_ON_FIRST_ELEMENT_NAME;
    static const std::string            RULE_ELEMENT_NAME;
    static const std::string            TYPE_ELEMENT_NAME;
    static const std::string            TRANSFORMATION_SET_VALUE_NAME;
    static const std::string            LOOKUP_DEFAULTACTION_ELEMENT_NAME;
    static const std::string            LOOKUP_LOOKUP_ELEMENT_NAME;
    static const std::string            LOOKUP_KEY_ATTRIBUTE_NAME;
    static const std::string            SOURCE_TERM_ELEMENT_NAME;
    static const std::string            REGEXP_ELEMENT_NAME;
    static const std::string            REGEXP_ATTRIBUTE_NAME;
    static const std::string            SEP_ATTRIBUTE_NAME;
    static const std::string            UNIQ_ATTRIBUTE_NAME;
};

inline bool AssignValue(EventPtr& e, const Vocabulary::Term& term, const std::string& value)
{
// Seems that not assigning empty values breaks functionality & tests
//  if (value.empty())      // New shortcut -- if empty value, don't assign
//    return true;

    try {
        e->set(term, value);
    } catch (...) {
        throw Transform::ValueAssignmentException(value);
    }
    return true;
}

class PION_PLATFORM_API TransformAssignValue
    : public Transform
{
    std::string                 m_tr_set_value;

public:

    TransformAssignValue(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <Value>escape(value)</Value>
        std::string val;
        if (! ConfigManager::getConfigOptionEmptyOk(VALUE_ELEMENT_NAME, val, config_ptr))
            throw MissingTransformField("Missing Value in TransformationAssignValue");
        m_tr_set_value = val;
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        return AssignValue(d, m_term, m_tr_set_value);
    }
};


class PION_PLATFORM_API TransformAssignTerm
    : public Transform
{
    Vocabulary::Term            m_src_term;

public:

    TransformAssignTerm(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <Term>src-term</Term>
        std::string term_id;
        if (! ConfigManager::getConfigOption(VALUE_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing Source-Term in TransformationAssignTerm");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid Source-Term in TransformationAssignTerm");
        m_src_term = v[term_ref];
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyCopied = false;
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            std::string str;
            AnyCopied |= AssignValue(d, m_term, s->write(str, ec->value, m_src_term));
        }
        return AnyCopied;   // true, if any were copied...
    }
};

class PION_PLATFORM_API TransformURLEncode
    : public Transform
{
    Vocabulary::Term            m_src_term;

public:

    TransformURLEncode(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <Term>src-term</Term>
        std::string term_id;
        if (! ConfigManager::getConfigOption(VALUE_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing Source-Term in TransformationURLEncode");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid Source-Term in TransformationURLEncode");
        m_src_term = v[term_ref];
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyCopied = false;
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            std::string str;
            AnyCopied |= AssignValue(d, m_term, algo::url_encode(s->write(str, ec->value, m_src_term)));
        }
        return AnyCopied;   // true, if any were copied...
    }
};

class PION_PLATFORM_API TransformURLDecode
    : public Transform
{
    Vocabulary::Term            m_src_term;

public:

    TransformURLDecode(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <Term>src-term</Term>
        std::string term_id;
        if (! ConfigManager::getConfigOption(VALUE_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing Source-Term in TransformationURLDecode");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid Source-Term in TransformationURLDecode");
        m_src_term = v[term_ref];
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyCopied = false;
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            std::string str;
            AnyCopied |= AssignValue(d, m_term, algo::url_decode(s->write(str, ec->value, m_src_term)));
        }
        return AnyCopied;   // true, if any were copied...
    }
};


class PION_PLATFORM_API TransformSplitTerm
    : public Transform
{
    Vocabulary::Term            m_src_term;

    boost::char_separator<char> m_sep;

public:

    TransformSplitTerm(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <Term>src-term</Term>
        std::string term_id;
        if (! ConfigManager::getConfigOption(VALUE_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing Source-Term in TransformationSplitTerm");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid Source-Term in TransformationSplitTerm");
        m_src_term = v[term_ref];
        std::string separator = ConfigManager::getAttribute(SEP_ATTRIBUTE_NAME, ConfigManager::findConfigNodeByName(VALUE_ELEMENT_NAME, config_ptr));
        if (separator.empty())
            throw MissingTransformField("Missing separator value in TransformationSplitTerm");
        m_sep = boost::char_separator<char>(separator.c_str());
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyCopied = false;
        typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        std::string str;
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            tokenizer str_tok(s->write(str, ec->value, m_src_term), m_sep);
            for (tokenizer::iterator tok_iter = str_tok.begin(); tok_iter != str_tok.end(); ++tok_iter)
                AnyCopied |= AssignValue(d, m_term, *tok_iter);
        }
        return AnyCopied;   // true, if any were copied...
    }
};

class PION_PLATFORM_API TransformJoinTerm
    : public Transform
{
    Vocabulary::Term            m_src_term;

    std::string                 m_separator;

    bool                        m_unique;

public:

    TransformJoinTerm(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term), m_unique(false)
    {
        // <Term>src-term</Term>
        std::string term_id;
        if (! ConfigManager::getConfigOption(VALUE_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing Source-Term in TransformationJoinTerm");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid Source-Term in TransformationJoinTerm");
        m_src_term = v[term_ref];
        m_separator = ConfigManager::getAttribute(SEP_ATTRIBUTE_NAME, ConfigManager::findConfigNodeByName(VALUE_ELEMENT_NAME, config_ptr));
        if (m_separator.empty())
            throw MissingTransformField("Missing separator value in TransformationSplitTerm");
        std::string uniq = ConfigManager::getAttribute(UNIQ_ATTRIBUTE_NAME, ConfigManager::findConfigNodeByName(VALUE_ELEMENT_NAME, config_ptr));
        if (uniq == "true")
            m_unique = true;
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyCopied = false;
        std::set<std::string> seen;
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        std::string result, str;
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            s->write(str, ec->value, m_src_term);
            if (!str.empty() && seen.find(str) == seen.end()) {
                if (!result.empty())
                    result += m_separator;
                result += str;
                if (m_unique)
                    seen.insert(str);
            }
        }
        if (!result.empty())
            AnyCopied |= AssignValue(d, m_term, result);
        return AnyCopied;   // true, if any were copied...
    }
};



class PION_PLATFORM_API TransformLookup
    : public Transform
{
    typedef PION_HASH_MAP<std::string, std::string, PION_HASH_STRING>   KVP;

    Vocabulary::Term            m_lookup_term;

    boost::regex                m_match;

    std::string                 m_format;

    enum { DEF_UNDEF, DEF_SRCTERM, DEF_OUTPUT, DEF_FIXED }
                                m_default;

    std::string                 m_fixed;

    KVP                         m_lookup;

    bool                        m_running;

public:

    TransformLookup(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        //  <LookupTerm>src-term</LookupTerm>
        std::string term_id;
        if (! ConfigManager::getConfigOption(LOOKUP_TERM_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing LookupTerm in TransformationAssignLookup");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid LookupTerm in TransformationAssignLookup");
        m_lookup_term = v[term_ref];
        //[opt]     <Match>escape(regexp)</Match>
        std::string val;
        if (ConfigManager::getConfigOptionEmptyOk(LOOKUP_MATCH_ELEMENT_NAME, val, config_ptr)) {
            try {
                m_match = val;
            } catch (...) {
                throw MissingTransformField("Invalid regular expression in TransformationLookup: " + val);
            }
        }
        //  [opt]       <Format>escape(format)</Format>
        m_format.clear();
        if (ConfigManager::getConfigOptionEmptyOk(LOOKUP_FORMAT_ELEMENT_NAME, val, config_ptr))
            m_format = val;

        //  [opt]       <DefaultAction>leave-undefined|src-term|output|fixedvalue</DefaultAction>
        m_default = DEF_UNDEF;
        if (ConfigManager::getConfigOption(LOOKUP_DEFAULTACTION_ELEMENT_NAME, val, config_ptr)) {
            if (val == "src-term")
                m_default = DEF_SRCTERM;
            else if (val == "output")
                m_default = DEF_OUTPUT;
            else if (val == "fixedvalue")
                m_default = DEF_FIXED;
        }
        //  [opt]       <DefaultValue>escape(text)</DefaultValue>
        m_fixed.clear();
        if (m_default == DEF_FIXED && ConfigManager::getConfigOptionEmptyOk(LOOKUP_DEFAULT_ELEMENT_NAME, val, config_ptr))
            m_fixed = val;
        //  [rpt/]      <Lookup key="escape(key)">escape(value)</Lookup>
        xmlNodePtr LookupNode = config_ptr;
        while ( (LookupNode = ConfigManager::findConfigNodeByName(LOOKUP_LOOKUP_ELEMENT_NAME, LookupNode)) != NULL) {
            // get the value (element content)
            xmlChar *xml_char_ptr = xmlNodeGetContent(LookupNode);
            if (xml_char_ptr == NULL || xml_char_ptr[0]=='\0') {
                if (xml_char_ptr != NULL)
                    xmlFree(xml_char_ptr);
                throw MissingTransformField("Missing Value in TransformationLookup");
            }
            const std::string val_str(reinterpret_cast<char*>(xml_char_ptr));
            xmlFree(xml_char_ptr);
            // next get the Term we want to map to
            xml_char_ptr = xmlGetProp(LookupNode, reinterpret_cast<const xmlChar*>(LOOKUP_KEY_ATTRIBUTE_NAME.c_str()));
            if (xml_char_ptr == NULL || xml_char_ptr[0]=='\0') {
                if (xml_char_ptr != NULL)
                    xmlFree(xml_char_ptr);
                throw MissingTransformField("Missing Key in TransformationLookup");
            }
            const std::string key_str(reinterpret_cast<char*>(xml_char_ptr));
            xmlFree(xml_char_ptr);
            if (m_lookup.find(key_str) != m_lookup.end())
                throw MissingTransformField("Duplicate Key in TransformationLookup");
            m_lookup[key_str] = val_str;
            LookupNode = LookupNode->next;
        }
        if (m_lookup.empty())
            throw MissingTransformField("No Key-Values in TransformationLookup");

        m_running = true;
    }

    virtual ~TransformLookup() {
        m_lookup.clear();
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        if (!m_running)
            return false;
        Event::ValuesRange values_range = s->equal_range(m_lookup_term.term_ref);
        Event::ConstIterator ec = values_range.first;
        // if ec == values_range.second ... source term was not found...
        bool AnyCopied = false;
        while (ec != values_range.second) {
            // Get the source term
            std::string str;
            s->write(str, ec->value, m_lookup_term);
            // If regex defined, do the regular expression, replacing the key value
            if (! m_match.empty()) {
                try {
                    str = boost::regex_replace(str, m_match, m_format, boost::format_all | boost::format_no_copy);
                } catch (...) {
                    // Get the source string again
                    str.clear();
                    s->write(str, ec->value, m_lookup_term);
                    // Not running anymore
                    m_running = false;
                    // Throw on this, to get an error message logged
                    throw RegexFailure("str=" + str + ", regex=" + m_match.str());
                }
            }
            // Find the value, using the key
            KVP::const_iterator i = m_lookup.find(str);
            if (i != m_lookup.end())    // Found: assign the lookup value
                AnyCopied |= AssignValue(d, m_term, i->second);
            else                        // Not found: perform default action
                switch (m_default) {
                    case DEF_UNDEF:     // Leave undefined, i.e. do nothing
                        break;
                    case DEF_SRCTERM:   // Re-get the original value, assign it
                        {
                            std::string str;
                            AnyCopied |= AssignValue(d, m_term, s->write(str, ec->value, m_lookup_term));
                        }
                        break;
                    case DEF_OUTPUT:    // Assign the regex output value
                        AnyCopied |= AssignValue(d, m_term, str);
                        break;
                    case DEF_FIXED:     // Assign the fixed value
                        AnyCopied |= AssignValue(d, m_term, m_fixed);
                        break;
                }
            ec++;           // repeat for all matching source terms
        }
        return AnyCopied;   // true, if any were copied...
    }
};

class PION_PLATFORM_API TransformRules
    : public Transform
{
    bool                                        m_short_circuit;

    std::vector<std::string>                    m_set_value;

    std::vector<Comparison *>                   m_comparison;

    std::vector<bool>                           m_running;

public:

    TransformRules(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        // <StopOnFirstMatch>true|false</StopOnFirstMatch>          -> DEFAULT: true
        m_short_circuit = false;
        std::string short_circuit_str;
        if (! ConfigManager::getConfigOption(RULES_STOP_ON_FIRST_ELEMENT_NAME, short_circuit_str, config_ptr))
            throw MissingTransformField("Missing StopOnFirstMatch in TransformationAssignRules");
        if (short_circuit_str == "true")
            m_short_circuit = true;

        //  [rpt]       <Rule>
        xmlNodePtr RuleNode = config_ptr;
        while ( (RuleNode = ConfigManager::findConfigNodeByName(RULE_ELEMENT_NAME, RuleNode)) != NULL)
        {
            //  <Term>src-term</Term>
            std::string term_id;
            if (! ConfigManager::getConfigOption(TERM_ELEMENT_NAME, term_id, RuleNode->children))
                throw MissingTransformField("Missing Source-Term in TransformationAssignRules");
            Vocabulary::TermRef term_ref = v.findTerm(term_id);
            if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
                throw MissingTransformField("Invalid Term in TransformationAssignRules");

            //  <Type>test-type</Type>
            std::string val;
            if (! ConfigManager::getConfigOption(TYPE_ELEMENT_NAME, val, RuleNode->children))
                throw MissingTransformField("Missing Value in TransformationAssignRules");
            Comparison::ComparisonType ctype = Comparison::parseComparisonType(val);

            //  <Value>escape(test-value)</Value>
            std::string value_str;
            if (Comparison::requiresValue(ctype))
                if (! ConfigManager::getConfigOptionEmptyOk(VALUE_ELEMENT_NAME, value_str, RuleNode->children))
                    throw MissingTransformField("Missing Value in TransformationAssignRules");

            Comparison *comp = new Comparison(v[term_ref]);
            comp->configure(ctype, value_str);
            m_comparison.push_back(comp);

            //  <SetValue>escape(set-value)</SetValue>
            val.clear();
            if (! ConfigManager::getConfigOptionEmptyOk(TRANSFORMATION_SET_VALUE_NAME, val, RuleNode->children))
                throw MissingTransformField("Missing SetValue in TransformationAssignRules");
            m_set_value.push_back(val);

            // Set running state
            m_running.push_back(true);

            RuleNode = RuleNode->next;
        }
    }

    virtual ~TransformRules() {
        for (unsigned int i = 0; i < m_comparison.size(); i++)
            delete m_comparison[i];
    }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyAssigned = false;
        // Loop through all TESTs, break out if any term successful on any test and short_circuit
        for (unsigned int i = 0; i < m_comparison.size(); i++)
            if (m_running[i]) {
                switch (m_comparison[i]->getType()) {
                    // We'll take out the cases where there might not be values to iterate through, and handle them individually.
                    case Comparison::TYPE_IS_DEFINED:
                        if (s->getType() == m_comparison[i]->getTerm().term_ref || s->isDefined(m_comparison[i]->getTerm().term_ref))
                            AnyAssigned |= AssignValue(d, m_term, m_set_value[i]);
                        break;
                    case Comparison::TYPE_TRUE:
                        AnyAssigned |= AssignValue(d, m_term, m_set_value[i]);
                        break;
                    case Comparison::TYPE_FALSE:
                        break;
                    case Comparison::TYPE_IS_NOT_DEFINED:
                        if (! (s->getType() == m_comparison[i]->getTerm().term_ref || s->isDefined(m_comparison[i]->getTerm().term_ref)))
                            AnyAssigned |= AssignValue(d, m_term, m_set_value[i]);
                        break;
                    default:
                        {
                            Event::ValuesRange values_range = s->equal_range(m_comparison[i]->getTerm().term_ref);
                            for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++)
                                try {
                                    Event::ConstIterator ec_past = ec;
                                    if (m_comparison[i]->evaluateRange(std::make_pair(ec, ++ec_past))) {
                                        if (m_comparison[i]->getType() == Comparison::TYPE_REGEX) {     // Only for POSITIVE regex...
                                            // Get the original value
                                            // For Regex... get the precompiled from Comparison
                                            // For Format... use the set_value
                                            std::string str;
                                            str = boost::u32regex_replace(s->write(str, ec->value, m_comparison[i]->getTerm()), m_comparison[i]->getRegex(),
                                                                        m_set_value[i], boost::format_all | boost::format_no_copy);
                                            // Assign the result
                                            AnyAssigned |= AssignValue(d, m_term, str);
                                        } else
                                            AnyAssigned |= AssignValue(d, m_term, m_set_value[i]);
                                    }
                                } catch (...) {
                                    // Get the original value again...
                                    std::string str;
                                    // This rule won't be running again...
                                    m_running[i] = false;
                                    // Throw on this, to get an error message logged
                                    throw RegexFailure("str=" + s->write(str, ec->value, m_comparison[i]->getTerm()) + ", regex=" + m_comparison[i]->getRegexStr());
                                }
                        }
                        break;
                }
                // If short_circuit AND any values were assigned -> don't go further in the chain
                if (m_short_circuit && AnyAssigned)
                    break;
            }
        return AnyAssigned;
    }
};

class PION_PLATFORM_API TransformRegex
    : public Transform
{
    Vocabulary::Term                            m_src_term;

    std::vector<std::string>                    m_format;

    std::vector<boost::u32regex>                m_regex;

    std::vector<std::string>                    m_regex_str;

    std::vector<bool>                           m_running;

public:

    TransformRegex(const Vocabulary& v, const Vocabulary::Term& term, const xmlNodePtr config_ptr)
        : Transform(v, term)
    {
        //  <SourceTerm>src-term</SourceTerm>
        std::string term_id;
        if (! ConfigManager::getConfigOption(SOURCE_TERM_ELEMENT_NAME, term_id, config_ptr))
            throw MissingTransformField("Missing SourceTerm in TransformationRegex");
        Vocabulary::TermRef term_ref = v.findTerm(term_id);
        if (term_ref == Vocabulary::UNDEFINED_TERM_REF)
            throw MissingTransformField("Invalid SourceTerm in TransformationRegex");
        m_src_term = v[term_ref];
        xmlNodePtr RegexNode = config_ptr;
        while ( (RegexNode = ConfigManager::findConfigNodeByName(REGEXP_ELEMENT_NAME, RegexNode)) != NULL) {
            // get the FORMAT (element content)
            xmlChar *xml_char_ptr = xmlNodeGetContent(RegexNode);
            std::string val;
            if (xml_char_ptr != NULL && xml_char_ptr[0] != '\0')
                val = reinterpret_cast<char*>(xml_char_ptr);
            if (xml_char_ptr != NULL) xmlFree(xml_char_ptr);
            m_format.push_back(val);
            // next get the Term we want to map to
            xml_char_ptr = xmlGetProp(RegexNode, reinterpret_cast<const xmlChar*>(REGEXP_ATTRIBUTE_NAME.c_str()));
            if (xml_char_ptr == NULL || xml_char_ptr[0]=='\0') {
                if (xml_char_ptr != NULL)
                    xmlFree(xml_char_ptr);
                throw MissingTransformField("Missing Regexp in TransformationRegex");
            }
            val = reinterpret_cast<char*>(xml_char_ptr);
            xmlFree(xml_char_ptr);
            boost::u32regex reg;
            try {
                reg = boost::make_u32regex(val);
            } catch (...) {
                throw MissingTransformField("Invalid regular expression in TransformationRegex");
            }
            m_regex.push_back(reg);
            m_regex_str.push_back(val);
            m_running.push_back(true);
            RegexNode = RegexNode->next;
        }
        if (m_regex.empty())
            throw MissingTransformField("No Regexp's in TransformationRegex");
    }

    virtual ~TransformRegex() { }

    virtual bool transform(EventPtr& d, const EventPtr& s)
    {
        bool AnyAssigned = false;
        // Iterate through all values from source term
        Event::ValuesRange values_range = s->equal_range(m_src_term.term_ref);
        for (Event::ConstIterator ec = values_range.first; ec != values_range.second; ec++) {
            // Take the original value from source term set
            std::string str;
            s->write(str, ec->value, m_src_term);
            // Run through all regexp's
            for (unsigned int i = 0; i < m_regex.size(); i++)
                if (m_running[i]) {
                    std::string res;
                    try {
                        res = boost::u32regex_replace(str, m_regex[i], m_format[i], boost::format_all | boost::format_no_copy);
                    } catch (...) {
                        // This rule won't be running again...
                        m_running[i] = false;
                        // Throw on this, to get an error message logged
                        throw RegexFailure("str=" + str + ", regex=" + m_regex_str[i]);
                    }
                    if (!res.empty())
                        str = res;
                }
            AnyAssigned |= AssignValue(d, m_term, str);
        }
        return AnyAssigned;
    }
};


template <class IteratorType>
inline bool HideCreditCardNumbers(IteratorType first, IteratorType last)
{
    // static regular expressions used to find and verify credit card numbers
    //
    // Visa: starts with "4", has 16 digits in blocks of 4
    // 4(?:[\s+-]?\d){15}
    //
    // MasterCard: starts with 51-55, has 16 digits in blocks of 4
    // 5[1-5](?:[\s+-]?\d){14}
    //
    // Amex: starts with 34 or 37, has 15 digits as 4-6-5
    // (?:34|37)(?:[\s+-]?\d){13}
    //
    // Discover: starts with 6011 or 65, has 16 digits in blocks of 4
    // (?:6011|65\d\d)(?:[\s+-]?\d){12}
    //
    // Diners: starts with 36, 38 or 300-305, all have 14 digits (blocks?)
    // (?:30[0-5]|36\d|38\d)(?:[\s+-]?\d){11}
    //
    // JCB: starts with 35, has 16 digits (blocks?)
    // 35(?:[\s+-]?\d){14}
    //
    // combined:
    // 4(?:[\s+-]?\d){15}|5[1-5](?:[\s+-]?\d){14}|(?:34|37)(?:[\s+-]?\d){13}|(?:6011|65\d\d)(?:[\s+-]?\d){12}|(?:30[0-5]|36\d|38\d)(?:[\s+-]?\d){11}|35(?:[\s+-]?\d){14}
    //
    // resources:
    // http://en.wikipedia.org/wiki/Credit_card_number
    // http://en.wikipedia.org/wiki/List_of_Bank_Identification_Numbers
    // http://www.regular-expressions.info/creditcard.html
    // 
    static const boost::regex FIND_CC_NUMBER_RX("\\b(?:4(?:[\\s+-]?\\d){15}|5[1-5](?:[\\s+-]?\\d){14}|(?:34|37)(?:[\\s+-]?\\d){13}|(?:6011|65\\d\\d)(?:[\\s+-]?\\d){12}|(?:30[0-5]|36\\d|38\\d)(?:[\\s+-]?\\d){11}|35(?:[\\s+-]?\\d){14})\\b");

    // variable used to store regex match information
    boost::match_results<IteratorType> results;
    
    // keeps track of whether or not we found a match
    bool found_match = false;

    // loop through the string looking for each possible match
    while (boost::regex_search(first, last, results, FIND_CC_NUMBER_RX)) {
        // basic verification succeeded
        found_match = true;
        
        // replace match string with X's
        for (IteratorType tmp_it = results[0].first; tmp_it != results[0].second; ++tmp_it) {
            *tmp_it = 'X';
        }

        // start searching for more cc numbers after end of current match
        // (no need to re-check earlier bytes)
        first = results[0].second;
    }
    
    return found_match;
}


inline bool HideCreditCardNumbers(Event& e, const Vocabulary::TermRef& term_ref)
{
    // keeps track of whether or not we found a match
    bool found_match = false;

    // get range of matching parameters within the event & iterate
    Event::ValuesRange values_range = e.equal_range(term_ref);
    for (Event::ConstIterator it = values_range.first; it != values_range.second; it++) {
        const Event::BlobType& b(boost::get<const Event::BlobType&>(it->value));
        char *first = const_cast<char*>(b.get());
        char *last = first + b.size();
        if (HideCreditCardNumbers(first, last)) {
            found_match = true;
        }
    }

    return found_match;
}


inline bool HideCreditCardNumbers(Event& e)
{
    // keeps track of whether or not we found a match
    bool found_match = false;

    // get range of matching parameters within the event & iterate
    for (Event::ConstIterator it = e.begin(); it != e.end(); it++) {
        if (boost::get<const Event::BlobType&>(& it->value)) {      // make sure it's BlobType
            const Event::BlobType& b(boost::get<const Event::BlobType&>(it->value));
            char *first = const_cast<char*>(b.get());
            char *last = first + b.size();
            if (HideCreditCardNumbers(first, last)) {
                found_match = true;
            }
        }
    }

    return found_match;
}


}   // end namespace platform
}   // end namespace pion


#endif

---