feat(super): Added Super agent and logs super agent targets (#35)

* feat(super): Added Super agent and logs super agent targets

* test(super): Deployer tests for puppet

* test(fossa): Remove puppet-module-gems

* test: Fix tests after updating PDK

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
 	"settings": {
 		"terminal.integrated.profiles.linux": {
 			"bash": {
-				"path": "bash",
+				"path": "bash"
 			}
 		}
 	},

.puppet-lint.rc

@@ -1,2 +1,2 @@
 --relative
---no-140chars-check
+--no-140chars-check

Gemfile

@@ -13,23 +13,31 @@ def location_for(place_or_version, fake_version = nil)
   end
 end
 
-ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments
-minor_version = ruby_version_segments[0..1].join('.')
-
 group :development do
-  gem "json", '= 2.0.4', require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
-  gem "json", '= 2.1.0', require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
-  gem "json", '= 2.3.0', require: false if Gem::Requirement.create(['>= 2.7.0', '< 2.8.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
-  gem "puppet-module-posix-default-r#{minor_version}", '~> 1.0', require: false, platforms: [:ruby]
-  gem "puppet-module-posix-dev-r#{minor_version}", '~> 1.0', require: false, platforms: [:ruby]
-  gem "puppet-module-win-default-r#{minor_version}", '~> 1.0', require: false, platforms: [:mswin, :mingw, :x64_mingw]
-  gem "puppet-module-win-dev-r#{minor_version}", '~> 1.0', require: false, platforms: [:mswin, :mingw, :x64_mingw]
-  gem 'concurrent-ruby', '< 1.2.0', require: false
-  gem 'rspec-puppet-utils', require: false
+  gem "json", '= 2.1.0',                           require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "json", '= 2.3.0',                           require: false if Gem::Requirement.create(['>= 2.7.0', '< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "json", '= 2.5.1',                           require: false if Gem::Requirement.create(['>= 3.0.0', '< 3.0.5']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "json", '= 2.6.1',                           require: false if Gem::Requirement.create(['>= 3.1.0', '< 3.1.3']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "json", '= 2.6.3',                           require: false if Gem::Requirement.create(['>= 3.2.0', '< 4.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "voxpupuli-puppet-lint-plugins", '~> 4.0',   require: false
+  gem "facterdb", '~> 1.18',                       require: false
+  gem "metadata-json-lint", '>= 2.0.2', '< 4.0.0', require: false
+  gem "puppetlabs_spec_helper", '~> 5.0',          require: false
+  gem "rspec-puppet-facts", '~> 2.0',              require: false
+  gem "codecov", '~> 0.2',                         require: false
+  gem "dependency_checker", '~> 0.2',              require: false
+  gem "parallel_tests", '= 3.12.1',                require: false
+  gem "pry", '~> 0.10',                            require: false
+  gem "simplecov-console", '~> 0.5',               require: false
+  gem "puppet-debugger", '~> 1.0',                 require: false
+  gem "rubocop", '= 1.6.1',                        require: false
+  gem "rubocop-performance", '= 1.9.1',            require: false
+  gem "rubocop-rspec", '= 2.0.1',                  require: false
+  gem "rb-readline", '= 0.5.5',                    require: false, platforms: [:mswin, :mingw, :x64_mingw]
 end
 group :system_tests do
-  gem "puppet-module-posix-system-r#{minor_version}", '~> 1.0', require: false, platforms: [:ruby]
-  gem "puppet-module-win-system-r#{minor_version}", '~> 1.0', require: false, platforms: [:mswin, :mingw, :x64_mingw]
+  gem "puppet_litmus", '< 1.0.0', require: false, platforms: [:ruby, :x64_mingw]
+  gem "serverspec", '~> 2.41',    require: false
 end
 
 puppet_version = ENV['PUPPET_GEM_VERSION']

Rakefile

@@ -127,9 +127,7 @@ def release_verify_branch
                         `"YbbdP"'   "Y888 `"YbbdP"'  88`YbbdP"'   
                                                      88           
                                                      88    
-
 You are attempting to release a version from  "#{branch}" - you must be on "main" to initiate the release process!
-
 EOM
 
   else
@@ -142,4 +140,4 @@ def release_tag_and_push
   sh "git commit -am 'Version bump to #{newVersion}' && git push origin main"
   Rake::Task['module:tag'].execute()
   sh "git push --tags"
-end
+end

lib/puppet/functions/newrelic_installer/targets_to_recipes.rb

@@ -18,7 +18,9 @@ def targets_to_recipes(targets)
       'logs' => 'logs-integration',
       'php' => 'php-agent-installer',
       'dotnet' => 'dotnet-agent-installer',
-      'nodejs' => 'node-agent-installer'
+      'nodejs' => 'node-agent-installer',
+      'super-agent' => 'super-agent',
+      'logs-super-agent' => 'logs-integration-super-agent'
     }
     requires_infrastructure_set = Set['logs']
 

metadata.json

@@ -89,7 +89,7 @@
       "version_requirement": ">= 6.21.0 < 8.0.0"
     }
   ],
-  "pdk-version": "2.5.0",
-  "template-url": "pdk-default#2.5.0",
-  "template-ref": "tags/2.5.0-0-g369d483"
+  "pdk-version": "2.7.1",
+  "template-url": "pdk-default#2.7.4",
+  "template-ref": "tags/2.7.4-0-g58edf57"
 }

spec/classes/install_spec.rb

@@ -151,4 +151,14 @@ def required_vars
       it { is_expected.to contain_exec('install newrelic instrumentation').with('command' => %r{(.*)--tag nr_deployed_by:puppet-install,some-tag:some-value,another-tag:another-value(.*)}) }
     end
   end
+  on_supported_os.each do |os, os_facts|
+    context "installed with trace verbosity on #{os}" do
+      let(:facts) { os_facts }
+      let(:params) do
+        required_vars.merge({ 'targets' => ['super-agent'], 'verbosity' => 'trace', 'proxy' => '' })
+      end
+
+      it { is_expected.to contain_exec('install newrelic instrumentation').with('command' => %r{(.*)--trace(.*)}) }
+    end
+  end
 end

spec/functions/targets_to_recipes_spec.rb

@@ -5,6 +5,8 @@
 describe 'newrelic_installer::targets_to_recipes' do
   # single valid recipe - infrastructure
   it { is_expected.to run.with_params(['infrastructure']).and_return('infrastructure-agent-installer') }
+  it { is_expected.to run.with_params(['super-agent']).and_return('super-agent') }
+  it { is_expected.to run.with_params(['logs-super-agent']).and_return('logs-integration-super-agent') }
   it { is_expected.to run.with_params(['php']).and_return('php-agent-installer') }
   it { is_expected.to run.with_params(['dotnet']).and_return('dotnet-agent-installer') }
   it { is_expected.to run.with_params(['nodejs']).and_return('node-agent-installer') }

spec/spec_helper.rb

@@ -6,7 +6,6 @@
 
 require 'puppetlabs_spec_helper/module_spec_helper'
 require 'rspec-puppet-facts'
-require 'rspec-puppet-utils'
 
 require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb'))
 

test/demo-deployer/setup/super/roles/configure/tasks/install-module/linux.yml

@@ -0,0 +1,26 @@
+---
+- name: Find locally-built module tarball
+  shell: find /home/puppet-install/pkg -type f -iname "newrelic-newrelic_installer*tar.gz" -print -quit
+  register: puppet_tarball_location
+
+- name: Install locally-built module from tarball
+  shell: puppet module install {{ puppet_tarball_location.stdout }}
+  register: command_output
+
+- name: ensure manifests directory exists
+  file:
+    path: $HOME/.puppetlabs/etc/code/manifests
+    state: directory
+
+- name: Copy site.pp to localhost
+  template:
+    src: site.pp.j2
+    dest: $HOME/.puppetlabs/etc/code/manifests/site.pp
+
+
+- name: Apply newrelic-newrelic_installer module
+  shell: sudo puppet apply --modulepath=$HOME/.puppetlabs/etc/code/modules $HOME/.puppetlabs/etc/code/manifests/site.pp
+  register: apply_output
+
+- debug:
+    msg: "{{ apply_output.stdout }}"

test/demo-deployer/setup/super/roles/configure/tasks/install-module/templates/site.pp.j2

@@ -0,0 +1,8 @@
+class { 'newrelic_installer::install':
+  targets               => ["super-agent"],
+  environment_variables => {
+    "NEW_RELIC_API_KEY"    => "{{ nr_api_key }}",
+    "NEW_RELIC_ACCOUNT_ID" => {{ nr_account_id }},
+    "NEW_RELIC_REGION"     => "{{ nr_region }}"
+  }
+}

test/demo-deployer/setup/super/roles/configure/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+# Install dependencies (git, puppet, pdk)
+- name: Install git
+  include_tasks: ../../../../packages/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/git/main.yml
+- name: Install puppet
+  include_tasks: ../../../../packages/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/puppet/main.yml
+- name: Install puppet-development-kit
+  include_tasks: ../../../../packages/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/puppet-development-kit/main.yml
+
+# Clone module repo and build locally
+- name: Clone puppet-install repository
+  ansible.builtin.git:
+    repo: https://github.com/newrelic/puppet-install.git
+    dest: /home/puppet-install
+  become: true
+- name: Build newrelic-newrelic_installer using PDK
+  shell: |
+    cd /home/puppet-install
+    /usr/local/bin/pdk build --force
+  become: true
+
+# Instrument infra recipe via module
+- name: Installing newrelic_installer module
+  include_tasks: install-module/linux.yml
+  vars:
+    nr_api_key: "{{ newrelic_personal_api_key }}"
+    nr_account_id: "{{ newrelic_account_id }}"
+    nr_region: "{{ newrelic_region|upper }}"

test/demo-deployer/targets/infrastructure/amzn2-infra.json

@@ -19,18 +19,14 @@
     "resources": [
       {
         "id": "amzn2_puppet-install_infra",
-        "resource_ids": [
-          "host1"
-        ],
+        "resource_ids": ["host1"],
         "provider": "newrelic",
         "local_source_path": "/mnt/deployer/puppet-install",
         "deploy_script_path": "/test/demo-deployer/setup/infrastructure/roles"
       },
       {
         "id": "recipeValidation",
-        "resource_ids": [
-          "host1"
-        ],
+        "resource_ids": ["host1"],
         "provider": "newrelic",
         "local_source_path": "/mnt/deployer/puppet-install",
         "deploy_script_path": "/test/demo-deployer/validate/roles",
@@ -40,4 +36,4 @@
       }
     ]
   }
-}
+}

test/demo-deployer/targets/super/amazonlinux2-super-agent-logs.json

@@ -0,0 +1,40 @@
+{
+  "global_tags": {
+    "owning_team": "virtuoso",
+    "Environment": "development",
+    "Department": "product",
+    "Product": "virtuoso"
+  },
+
+  "resources": [
+    {
+      "id": "host1",
+      "provider": "aws",
+      "ami_name": "amzn2-ami-hvm-2.0.????????.?-x86_64-gp2",
+      "type": "ec2",
+      "size": "t3.micro"
+    }
+  ],
+
+  "instrumentations": {
+    "resources": [
+      {
+        "id": "amzn2_puppet-install_infra",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/setup/super/roles"
+      },
+      {
+        "id": "recipeValidation",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/validate/roles",
+        "params": {
+          "nrql_query": "select count(*) from SystemSample where hostname like '%HOSTNAME%' since 10 minutes ago"
+        }
+      }
+    ]
+  }
+}

test/demo-deployer/targets/super/amazonlinux2023-super-agent-logs.json

@@ -0,0 +1,41 @@
+{
+  "global_tags": {
+    "owning_team": "virtuoso",
+    "Environment": "development",
+    "Department": "product",
+    "Product": "virtuoso"
+  },
+
+  "resources": [
+    {
+      "id": "host1",
+      "provider": "aws",
+      "type": "ec2",
+      "size": "t3.small",
+      "ami_name": "al2023-ami-2023.*-x86_64",
+      "user_name": "ec2-user"
+    }
+  ],
+
+  "instrumentations": {
+    "resources": [
+      {
+        "id": "amzn2_puppet-install_infra",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/setup/super/roles"
+      },
+      {
+        "id": "recipeValidation",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/validate/roles",
+        "params": {
+          "nrql_query": "select count(*) from SystemSample where hostname like '%HOSTNAME%' since 10 minutes ago"
+        }
+      }
+    ]
+  }
+}

test/demo-deployer/targets/super/awslinux23-super-agent.json

@@ -0,0 +1,40 @@
+{
+  "global_tags": {
+    "owning_team": "virtuoso",
+    "Environment": "development",
+    "Department": "product",
+    "Product": "virtuoso"
+  },
+  "resources": [
+    {
+      "id": "host1",
+      "provider": "aws",
+      "ami_name": "al2023-ami-2023.*-x86_64",
+      "type": "ec2",
+      "size": "t3.nano",
+      "user_name": "ec2-user"
+    }
+  ],
+  "instrumentations": {
+    "resources": [
+    "resources": [
+      {
+        "id": "amzn2_puppet-install_infra",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/setup/super/roles"
+      },
+      {
+        "id": "recipeValidation",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/validate/roles",
+        "params": {
+          "nrql_query": "select count(*) from SystemSample where hostname like '%HOSTNAME%' since 10 minutes ago"
+        }
+      }
+    ]
+  }
+}

test/demo-deployer/targets/super/ubuntu20-super-agent.json

@@ -0,0 +1,41 @@
+{
+  "global_tags": {
+    "owning_team": "virtuoso",
+    "Environment": "development",
+    "Department": "product",
+    "Product": "virtuoso"
+  },
+
+  "resources": [
+    {
+      "id": "host1",
+      "provider": "aws",
+      "type": "ec2",
+      "size": "t3.nano",
+      "ami_name": "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-2023*",
+      "user_name": "ubuntu"
+    }
+  ],
+
+  "instrumentations": {
+    "resources": [
+      {
+        "id": "amzn2_puppet-install_infra",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/setup/super/roles"
+      },
+      {
+        "id": "recipeValidation",
+        "resource_ids": ["host1"],
+        "provider": "newrelic",
+        "local_source_path": "/mnt/deployer/puppet-install",
+        "deploy_script_path": "/test/demo-deployer/validate/roles",
+        "params": {
+          "nrql_query": "select count(*) from SystemSample where hostname like '%HOSTNAME%' since 10 minutes ago"
+        }
+      }
+    ]
+  }
+}