Merge pull request #915 from vespa-engine/thomasht86/dynamic-xml-crea…

…tion

(feat)First step towards full xml-configuration support

.github/scripts/copy_sample_files.py

@@ -0,0 +1,81 @@
+import argparse
+import tempfile
+import os
+import shutil
+from git import Repo
+from pathlib import Path
+
+
+def clone_repo_shallow(repo_url, temp_dir):
+    """Clone the given GitHub repository to a temporary directory with depth 1 and without blobs."""
+    repo = Repo.clone_from(repo_url, temp_dir, depth=1, no_checkout=True)
+    return repo
+
+
+def sparse_checkout(repo, paths):
+    """Perform a sparse checkout of specified paths."""
+    git_dir = repo.git_dir
+    sparse_checkout_file = Path(git_dir) / "info" / "sparse-checkout"
+    sparse_checkout_file.parent.mkdir(parents=True, exist_ok=True)
+
+    print("Sparse checkout paths:")
+    with sparse_checkout_file.open("w") as f:
+        for path in paths:
+            print(f"Adding to sparse checkout: {path}")
+            f.write(f"{path}\n")
+
+    repo.git.config("core.sparseCheckout", "true")
+    repo.git.checkout()
+
+
+def copy_files_preserving_structure(temp_dir, output_dir, file_name, subfolder):
+    """Copy all files with a specific name to a given subfolder within the output directory, preserving the original directory structure."""
+    dest_base_dir = os.path.join(output_dir, subfolder)
+    os.makedirs(dest_base_dir, exist_ok=True)
+
+    print(f"Searching for {file_name} in {temp_dir}...")
+    for root, dirs, files in os.walk(temp_dir):
+        if file_name in files:
+            src_file = os.path.join(root, file_name)
+            relative_path = os.path.relpath(root, temp_dir)
+            dest_dir = os.path.join(dest_base_dir, relative_path)
+            os.makedirs(dest_dir, exist_ok=True)
+            shutil.copy(src_file, dest_dir)
+            print(f"Copied {src_file} to {dest_dir}")
+
+
+def main(output_dir):
+    repo_url = "https://github.com/vespa-engine/sample-apps/"
+
+    # Create a temporary directory to clone the repo
+    with tempfile.TemporaryDirectory() as temp_dir:
+        print(f"Cloning repo to temporary directory {temp_dir}...")
+
+        # Shallow clone the repo without blobs
+        repo = clone_repo_shallow(repo_url, temp_dir)
+
+        # Specify the files you want to checkout
+        paths = ["**/services.xml", "**/validation-overrides.xml", "**/hosts.xml"]
+        sparse_checkout(repo, paths)
+
+        # Copy files to respective folders in the output directory, preserving directory structure
+        copy_files_preserving_structure(
+            temp_dir, output_dir, "services.xml", "services"
+        )
+        copy_files_preserving_structure(
+            temp_dir, output_dir, "validation-overrides.xml", "validations"
+        )
+        copy_files_preserving_structure(temp_dir, output_dir, "hosts.xml", "hosts")
+        print(f"Files copied to {output_dir} successfully.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Copy specific XML files from a GitHub repo to an output directory, preserving directory structure."
+    )
+    parser.add_argument(
+        "output_dir", type=str, help="The directory to output the copied files."
+    )
+    args = parser.parse_args()
+
+    main(args.output_dir)

pyproject.toml

@@ -22,6 +22,8 @@ dependencies = [
     "tenacity>=8.4.1",
     "typing_extensions",
     "python-dateutil",
+    "fastcore>=1.7.8",
+    "lxml",
 ]
 
 requires-python = ">=3.8"
@@ -135,7 +137,7 @@ vespa = ["py.typed", "templates/*"]
 
 [tool.ruff.lint]
 select = ["E4", "E7", "E9", "F"]
-ignore = []
+ignore = ["F822", "F405", "F403"]
 fixable = ["ALL"]
 unfixable = []
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

tests/testfiles/hosts/examples/document-processing/src/main/application/hosts.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="localhost">
+    <alias>node1</alias>
+  </host>
+</hosts>
+

tests/testfiles/hosts/examples/generic-request-processing/src/main/application/hosts.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="localhost">
+    <alias>node1</alias>
+  </host>
+</hosts>

tests/testfiles/hosts/examples/http-api-using-request-handlers-and-processors/src/main/application/hosts.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="localhost">
+    <alias>node1</alias>
+  </host>
+</hosts>

tests/testfiles/hosts/examples/joins/src/main/application/hosts.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="localhost">
+    <alias>node1</alias>
+  </host>
+</hosts>
+

tests/testfiles/hosts/examples/multiple-bundles/src/main/application/hosts.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="localhost">
+    <alias>node1</alias>
+  </host>
+</hosts>

tests/testfiles/hosts/examples/operations/multinode-HA/gke/hosts.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<hosts>
+  <host name='vespa-configserver-0.vespa-internal.default.svc.cluster.local'>
+    <alias>node0</alias>
+  </host>
+  <host name='vespa-configserver-1.vespa-internal.default.svc.cluster.local'>
+    <alias>node1</alias>
+  </host>
+  <host name='vespa-configserver-2.vespa-internal.default.svc.cluster.local'>
+    <alias>node2</alias>
+  </host>
+  <host name='vespa-admin-0.vespa-internal.default.svc.cluster.local'>
+    <alias>node3</alias>
+  </host>
+  <host name='vespa-feed-container-0.vespa-internal.default.svc.cluster.local'>
+    <alias>node4</alias>
+  </host>
+  <host name='vespa-feed-container-1.vespa-internal.default.svc.cluster.local'>
+    <alias>node5</alias>
+  </host>
+  <host name='vespa-query-container-0.vespa-internal.default.svc.cluster.local'>
+    <alias>node6</alias>
+  </host>
+  <host name='vespa-query-container-1.vespa-internal.default.svc.cluster.local'>
+    <alias>node7</alias>
+  </host>
+  <host name='vespa-content-0.vespa-internal.default.svc.cluster.local'>
+    <alias>node8</alias>
+  </host>
+  <host name='vespa-content-1.vespa-internal.default.svc.cluster.local'>
+    <alias>node9</alias>
+  </host>
+</hosts>

tests/testfiles/hosts/examples/operations/multinode-HA/hosts.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="node0.vespanet">
+    <alias>node0</alias>
+  </host>
+  <host name="node1.vespanet">
+    <alias>node1</alias>
+  </host>
+  <host name="node2.vespanet">
+    <alias>node2</alias>
+  </host>
+
+  <host name="node3.vespanet">
+    <alias>node3</alias>
+  </host>
+
+  <host name="node4.vespanet">
+    <alias>node4</alias>
+  </host>
+  <host name="node5.vespanet">
+    <alias>node5</alias>
+  </host>
+
+  <host name="node6.vespanet">
+    <alias>node6</alias>
+  </host>
+  <host name="node7.vespanet">
+    <alias>node7</alias>
+  </host>
+
+  <host name="node8.vespanet">
+    <alias>node8</alias>
+  </host>
+  <host name="node9.vespanet">
+    <alias>node9</alias>
+  </host>
+</hosts>

tests/testfiles/hosts/examples/operations/multinode/hosts.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="node0.vespanet">
+    <alias>node0</alias>
+  </host>
+  <host name="node1.vespanet">
+    <alias>node1</alias>
+  </host>
+  <host name="node2.vespanet">
+    <alias>node2</alias>
+  </host>
+</hosts>

tests/testfiles/relaxng/admin.rnc

@@ -0,0 +1,132 @@
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+Admin = AdminV2 | AdminV3 | AdminV4
+
+AdminV2 =
+ element admin {
+   attribute version { "2.0" } &
+   element adminserver { service.attlist }? &
+   GenericConfig* &
+   LogServer? &
+   ConfigServers? &
+   AdminSlobroks? &
+   AdminMonitoring? &
+   Metrics? &
+   ClusterControllers? &
+   LoggingSpecs? &
+   LogForwarding?
+ }
+
+AdminV3 =
+  element admin {
+    attribute version { "3.0"  } &
+    GenericConfig* &
+    Nodes
+  }
+
+AdminV4 =
+  element admin {
+    attribute version { "4.0"  } &
+    AdminV4Slobroks? &
+    AdminV4LogServers? &
+    GenericConfig* &
+    AdminMonitoring? &
+    Metrics? &
+    LoggingSpecs? &
+    LogForwarding?
+  }
+
+AdminV4Slobroks =
+  element slobroks {
+    OptionalDedicatedNodes
+  }
+
+AdminV4LogServers =
+  element logservers {
+    OptionalDedicatedNodes
+  }
+
+AdminSlobroks =
+   element slobroks {
+      element slobrok {
+         service.attlist &
+         attribute index { xsd:nonNegativeInteger }?
+      }+
+   }
+
+AdminMonitoring =
+   element monitoring {
+      attribute interval { xsd:int }?,
+      attribute systemname { xsd:string }?
+   }
+
+ConfigServer = element configserver {
+   service.attlist
+}
+
+ConfigServers = element configservers {
+   ConfigServer+
+}
+
+LogServer = element logserver {
+   service.attlist
+}
+
+Metrics = element metrics {
+    element consumer {
+        attribute id { xsd:Name } &
+        element metric-set { attribute id { xsd:Name } }* &
+        element metric {
+            attribute id { xsd:Name } &
+            attribute display-name { xsd:Name }?
+        }* &
+        Cloudwatch?
+    }+
+}
+
+Cloudwatch = element cloudwatch {
+    attribute region { xsd:Name } &
+    attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } &
+    (
+      element credentials {
+            attribute access-key-name { xsd:Name } &
+            attribute secret-key-name { xsd:Name }
+      }
+      |
+      element shared-credentials {
+          attribute file { string } &
+          attribute profile { xsd:Name }?
+      }
+    )?
+}
+
+ClusterControllers = element cluster-controllers {
+    attribute standalone-zookeeper { xsd:string  }? &
+    element cluster-controller {
+       service.attlist
+    }+
+}
+
+LogForwarding = element logforwarding {
+    attribute include-admin { xsd:boolean }? &
+    element splunk {
+        attribute splunk-home { xsd:string }? &
+        attribute deployment-server { xsd:string } &
+        attribute client-name { xsd:string } &
+        attribute phone-home-interval { xsd:positiveInteger }? &
+        attribute role { xsd:string }?
+    }
+}
+
+LoggingSpecs = element logging {
+    (
+    element class {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    |
+    element package {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    )*
+}