Merge pull request #23934 from vespa-engine/vekterli/factor-out-xml-s…

…tring-escaping

Factor out XML string escaping and use for internal legacy status pages [run-systemtest]

storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp

@@ -14,12 +14,15 @@
 #include <vespa/storageapi/message/stat.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/string_escape.h>
 #include <xxhash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.filestor.handler.impl");
 
 using document::BucketSpace;
+using vespalib::xml_attribute_escaped;
+using vespalib::xml_content_escaped;
 
 namespace storage {
 
@@ -1338,8 +1341,8 @@ FileStorHandlerImpl::Stripe::dumpQueueHtml(std::ostream & os) const
 
     const PriorityIdx& idx = bmi::get<1>(*_queue);
     for (const auto & entry : idx) {
-        os << "<li>" << entry._command->toString() << " (priority: "
-           << (int)entry._command->getPriority() << ")</li>\n";
+        os << "<li>" << xml_content_escaped(entry._command->toString()) << " (priority: "
+           << static_cast<int>(entry._command->getPriority()) << ")</li>\n";
     }
 }
 
@@ -1379,8 +1382,9 @@ FileStorHandlerImpl::Stripe::dumpQueue(std::ostream & os) const
 
     const PriorityIdx& idx = bmi::get<1>(*_queue);
     for (const auto & entry : idx) {
-        os << entry._bucket.getBucketId() << ": " << entry._command->toString() << " (priority: "
-           << (int)entry._command->getPriority() << ")\n";
+        os << entry._bucket.getBucketId() << ": "
+           << xml_content_escaped(entry._command->toString())
+           << " (priority: " << static_cast<int>(entry._command->getPriority()) << ")\n";
     }
 }
 

storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp

@@ -23,6 +23,7 @@
 #include <vespa/vespalib/util/cpu_usage.h>
 #include <vespa/vespalib/util/idestructorcallback.h>
 #include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/string_escape.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/subscription/configuri.h>
 #include <vespa/config/helper/configfetcher.hpp>
@@ -887,16 +888,17 @@ void FileStorManager::onFlush(bool downwards)
 void
 FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath& path) const
 {
-    bool showStatus = !path.hasAttribute("thread");
-    bool verbose = path.hasAttribute("verbose");
+    using vespalib::xml_attribute_escaped;
 
-        // Print menu
+    bool showStatus = !path.hasAttribute("thread");
+    bool verbose    = path.hasAttribute("verbose");
+    // Print menu
     out << "<font size=\"-1\">[ <a href=\"/\">Back to top</a>"
         << " | <a href=\"?" << (verbose ? "verbose" : "")
         << "\">Main filestor manager status page</a>"
         << " | <a href=\"?" << (verbose ? "notverbose" : "verbose");
     if (!showStatus) {
-        out << "&thread=" << path.get("thread", std::string(""));
+        out << "&thread=" << xml_attribute_escaped(path.get("thread", std::string("")));
     }
     out << "\">" << (verbose ? "Less verbose" : "More verbose") << "</a>\n"
         << " ]</font><br><br>\n";

storage/src/vespa/storage/storageserver/mergethrottler.cpp

@@ -11,6 +11,7 @@
 #include <vespa/config/helper/configfetcher.hpp>
 #include <vespa/config/subscription/configuri.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/string_escape.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <cassert>
 
@@ -1313,6 +1314,8 @@ void
 MergeThrottler::reportHtmlStatus(std::ostream& out,
                                  const framework::HttpUrlPath&) const
 {
+    using vespalib::xml_content_escaped;
+
     std::lock_guard lock(_stateLock);
     if (_use_dynamic_throttling) {
         out << "<p>Dynamic throttle policy; window size min/max: ["
@@ -1333,7 +1336,7 @@ MergeThrottler::reportHtmlStatus(std::ostream& out,
     if (!_merges.empty()) {
         out << "<ul>\n";
         for (auto& m : _merges) {
-            out << "<li>" << m.second.getMergeCmdString();
+            out << "<li>" << xml_content_escaped(m.second.getMergeCmdString());
             if (m.second.isExecutingLocally()) {
                 out << " <strong>(";
                 if (m.second.isInCycle()) {
@@ -1364,7 +1367,7 @@ MergeThrottler::reportHtmlStatus(std::ostream& out,
             // The queue always owns its messages, thus this is safe
             out << "<li>Pri "
                 << static_cast<unsigned int>(qm._msg->getPriority())
-                << ": " << *qm._msg;
+                << ": " << xml_content_escaped(qm._msg->toString());
             out << "</li>\n";
         }
         out << "</ol>\n";

storage/src/vespa/storage/storageserver/statemanager.cpp

@@ -9,9 +9,9 @@
 #include <vespa/storageapi/messageapi/storagemessage.h>
 #include <vespa/vdslib/state/cluster_state_bundle.h>
 #include <vespa/vdslib/state/clusterstate.h>
-#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/string_escape.h>
 #include <vespa/vespalib/util/stringfmt.h>
 
 #include <fstream>
@@ -99,27 +99,28 @@ void
 StateManager::reportHtmlStatus(std::ostream& out,
                                const framework::HttpUrlPath& path) const
 {
+    using vespalib::xml_content_escaped;
     (void) path;
     {
         std::lock_guard lock(_stateLock);
-        const auto &baseLineClusterState = _systemState->getBaselineClusterState();
+        const auto& baseLineClusterState = _systemState->getBaselineClusterState();
         out << "<h1>Current system state</h1>\n"
-            << "<code>" << baseLineClusterState->toString(true) << "</code>\n"
+            << "<code>" << xml_content_escaped(baseLineClusterState->toString(true)) << "</code>\n"
             << "<h1>Current node state</h1>\n"
             << "<code>" << baseLineClusterState->getNodeState(lib::Node(
                         _component.getNodeType(), _component.getIndex())
                                                      ).toString(true)
             << "</code>\n"
             << "<h1>Reported node state</h1>\n"
-            << "<code>" << _nodeState->toString(true) << "</code>\n"
+            << "<code>" << xml_content_escaped(_nodeState->toString(true)) << "</code>\n"
             << "<h1>Pending state requests</h1>\n"
             << _queuedStateRequests.size() << "\n"
             << "<h1>System state history</h1>\n"
             << "<table border=\"1\"><tr>"
             << "<th>Received at time</th><th>State</th></tr>\n";
         for (auto it = _systemStateHistory.rbegin(); it != _systemStateHistory.rend(); ++it) {
             out << "<tr><td>" << it->first << "</td><td>"
-                << *it->second->getBaselineClusterState() << "</td></tr>\n";
+                << xml_content_escaped(it->second->getBaselineClusterState()->toString()) << "</td></tr>\n";
         }
         out << "</table>\n";
     }

storage/src/vespa/storage/visiting/visitor.cpp

@@ -2,15 +2,15 @@
 
 #include "visitor.h"
 #include "visitormetrics.h"
+#include <vespa/document/select/node.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/documentapi/messagebus/messages/visitor.h>
 #include <vespa/persistence/spi/docentry.h>
-#include <vespa/storageframework/generic/clock/timer.h>
 #include <vespa/storageapi/message/datagram.h>
+#include <vespa/storageframework/generic/clock/timer.h>
 #include <vespa/storage/persistence/messages.h>
-#include <vespa/documentapi/messagebus/messages/visitor.h>
-#include <vespa/document/select/node.h>
-#include <vespa/document/fieldset/fieldsets.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/string_escape.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <unordered_map>
 #include <sstream>
@@ -932,10 +932,12 @@ Visitor::continueVisitor()
 void
 Visitor::getStatus(std::ostream& out, bool verbose) const
 {
+    using vespalib::xml_content_escaped;
+
     out << "<table border=\"1\"><tr><td>Property</td><td>Value</td></tr>\n";
 
     out << "<tr><td>Visitor id</td><td>" << _visitorId << "</td></tr>\n";
-    out << "<tr><td>Visitor name</td><td>" << _id << "</td></tr>\n";
+    out << "<tr><td>Visitor name</td><td>" << xml_content_escaped(_id) << "</td></tr>\n";
 
     out << "<tr><td>Number of buckets to visit</td><td>" << _buckets.size()
         << "</td></tr>\n";
@@ -953,7 +955,7 @@ Visitor::getStatus(std::ostream& out, bool verbose) const
         << "</td></tr>\n";
 
     out << "<tr><td>Current status</td><td>"
-        << _result << "</td></tr>\n";
+        << xml_content_escaped(_result.toString()) << "</td></tr>\n";
 
     out << "<tr><td>Failed</td><td>" << (failed() ? "true" : "false")
         << "</td></tr>\n";
@@ -973,28 +975,28 @@ Visitor::getStatus(std::ostream& out, bool verbose) const
         out << "<tr><td>Called completed visitor</td><td>"
             << (_calledCompletedVisitor ? "true" : "false") << "</td></tr>\n";
         out << "<tr><td>Visiting fields</td><td>"
-            << _visitorOptions._fieldSet
+            << xml_content_escaped(_visitorOptions._fieldSet)
             << "</td></tr>\n";
         out << "<tr><td>Visiting removes</td><td>"
             << (_visitorOptions._visitRemoves ? "true" : "false")
             << "</td></tr>\n";
         out << "<tr><td>Control destination</td><td>";
         if (_controlDestination.get()) {
-            out << _controlDestination->toString();
+            out << xml_content_escaped(_controlDestination->toString());
         } else {
             out << "nil";
         }
         out << "</td></tr>\n";
         out << "<tr><td>Data destination</td><td>";
         if (_dataDestination.get()) {
-            out << _dataDestination->toString();
+            out << xml_content_escaped(_dataDestination->toString());
         } else {
             out << "nil";
         }
         out << "</td></tr>\n";
         out << "<tr><td>Document selection</td><td>";
         if (_documentSelection.get()) {
-            out << *_documentSelection;
+            out << xml_content_escaped(_documentSelection->toString());
         } else {
             out << "nil";
         }
@@ -1052,7 +1054,7 @@ Visitor::getStatus(std::ostream& out, bool verbose) const
     for (auto& idAndMeta : _visitorTarget._messageMeta) {
         const VisitorTarget::MessageMeta& meta(idAndMeta.second);
         out << "Message #" << idAndMeta.first << " <b>"
-            << meta.messageText << "</b> ";
+            << xml_content_escaped(meta.messageText) << "</b> ";
         if (meta.retryCount > 0) {
             out << "Retried " << meta.retryCount << " times. ";
         }

storage/src/vespa/storage/visiting/visitormanager.cpp

@@ -7,11 +7,12 @@
 #include "testvisitor.h"
 #include "recoveryvisitor.h"
 #include "reindexing_visitor.h"
-#include <vespa/storage/common/statusmessages.h>
 #include <vespa/config/subscription/configuri.h>
 #include <vespa/config/common/exceptions.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/helper/configfetcher.hpp>
+#include <vespa/storage/common/statusmessages.h>
+#include <vespa/vespalib/util/string_escape.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <cassert>
 
 #include <vespa/log/log.h>
@@ -557,6 +558,9 @@ void
 VisitorManager::reportHtmlStatus(std::ostream& out,
                                  const framework::HttpUrlPath& path) const
 {
+    using vespalib::xml_attribute_escaped;
+    using vespalib::xml_content_escaped;
+
     bool showStatus = !path.hasAttribute("visitor");
     bool verbose = path.hasAttribute("verbose");
     bool showAll = path.hasAttribute("allvisitors");
@@ -568,7 +572,7 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
         << " | <a href=\"?allvisitors" << (verbose ? "&verbose" : "")
         << "\">Show all visitors</a>"
         << " | <a href=\"?" << (verbose ? "notverbose" : "verbose");
-    if (!showStatus) out << "&visitor=" << path.get("visitor", std::string(""));
+    if (!showStatus) out << "&visitor=" << xml_attribute_escaped(path.get("visitor", std::string("")));
     if (showAll) out << "&allvisitors";
     out << "\">" << (verbose ? "Less verbose" : "More verbose") << "</a>\n"
         << " ]</font><br><br>\n";
@@ -585,7 +589,8 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
                     out << " none";
                 } else {
                     for (const auto& id_and_visitor : _visitorThread[i].second) {
-                        out << " " << id_and_visitor.second << " (" << id_and_visitor.first << ")";
+                        out << " " << xml_content_escaped(id_and_visitor.second)
+                            << " (" << id_and_visitor.first << ")";
                     }
                 }
                 out << "<br>\n";
@@ -596,7 +601,8 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
             for (const auto& enqueued : _visitorQueue) {
                 auto& cmd = enqueued._command;
                 assert(cmd);
-                out << "<li>" << cmd->getInstanceId() << " - "
+                out << "<li>"
+                    << xml_content_escaped(cmd->getInstanceId()) << " - "
                     << vespalib::count_ms(cmd->getQueueTimeout()) << ", remaining timeout "
                     << vespalib::count_ms(enqueued._deadline - now) << " ms\n";
             }
@@ -619,7 +625,7 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
                         << "<td>" << entry.second.id << "</td>"
                         << "<td>" << entry.second.timestamp << "</td>"
                         << "<td>" << vespalib::count_ms(entry.second.timeout) << "</td>"
-                        << "<td>" << entry.second.destination << "</td>"
+                        << "<td>" << xml_content_escaped(entry.second.destination) << "</td>"
                         << "</tr>\n";
                 }
                 out << "</table>\n";
@@ -632,13 +638,13 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
             << ", variable = " << _maxVariableConcurrentVisitors
             << ", waiting visitors " << _visitorQueue.size() << "<br>\n";
     }
-        // Only one can access status at a time as _statusRequest only holds
-        // answers from one request at a time
+    // Only one can access status at a time as _statusRequest only holds
+    // answers from one request at a time
     std::unique_lock sync(_statusLock);
-        // Send all subrequests
+    // Send all sub-requests
     uint32_t parts = _visitorThread.size();
     for (uint32_t i=0; i<parts; ++i) {
-        std::shared_ptr<RequestStatusPage> cmd(new RequestStatusPage(path));
+        auto cmd = std::make_shared<RequestStatusPage>(path);
         std::ostringstream token;
         token << "Visitor thread " << i;
         cmd->setSortToken(token.str());
@@ -648,7 +654,7 @@ VisitorManager::reportHtmlStatus(std::ostream& out,
     _statusCond.wait(sync, [&]() { return (_statusRequest.size() >= parts);});
     std::sort(_statusRequest.begin(), _statusRequest.end(), StatusReqSorter());
 
-        // Create output
+    // Create output
     for (uint32_t i=0; i<_statusRequest.size(); ++i) {
         out << "<h2>" << _statusRequest[i]->getSortToken()
             << "</h2>\n" << _statusRequest[i]->getStatus() << "\n";

storage/src/vespa/storage/visiting/visitorthread.cpp

@@ -2,18 +2,16 @@
 
 #include "visitorthread.h"
 #include "messages.h"
+#include <vespa/document/base/exceptions.h>
 #include <vespa/document/select/bodyfielddetector.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/messagebus/rpcmessagebus.h>
+#include <vespa/storageapi/message/datagram.h>
 #include <vespa/storage/common/statusmessages.h>
 #include <vespa/storage/config/config-stor-server.h>
-#include <vespa/storageapi/message/datagram.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/document/base/exceptions.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
 #include <locale>
-#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.thread");

vespalib/CMakeLists.txt

@@ -189,6 +189,7 @@ vespa_define_module(
     src/tests/util/rcuvector
     src/tests/util/reusable_set
     src/tests/util/size_literals
+    src/tests/util/string_escape
     src/tests/valgrind
     src/tests/visit_ranges
     src/tests/invokeservice

vespalib/src/tests/util/string_escape/CMakeLists.txt

@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_util_string_escape_test_app TEST
+    SOURCES
+    string_escape_test.cpp
+    DEPENDS
+    vespalib
+    GTest::GTest
+)
+vespa_add_test(NAME vespalib_util_string_escape_test_app COMMAND vespalib_util_string_escape_test_app)