From 812279cee023df8552b32c640f7aaf15ccf9512a Mon Sep 17 00:00:00 2001 From: Serge ALEXANDRE Date: Sat, 1 Jul 2017 11:39:59 +0200 Subject: [PATCH] Document encryption feature --- bin/hadeploy | 2 +- docsite/docs/gettingstarted.md | 10 +- docsite/docs/more/encrypted_values.md | 118 ++++++++++++++++++ docsite/docs/more/release_notes.md | 9 +- docsite/docs/more/under_the_hood.md | 2 +- .../ansible_inventories.md | 46 +++++++ .../ansible_inventory_files.md | 9 +- .../plugins_reference/ranger/ranger_relay.md | 26 +++- docsite/mkdocs.yml | 3 + 9 files changed, 215 insertions(+), 10 deletions(-) create mode 100644 docsite/docs/more/encrypted_values.md create mode 100644 docsite/docs/plugins_reference/ansible_inventories/ansible_inventories.md diff --git a/bin/hadeploy b/bin/hadeploy index c6c9d18..da80393 100755 --- a/bin/hadeploy +++ b/bin/hadeploy @@ -20,7 +20,7 @@ CWD=$(pwd) function usage { - echo 'hadeploy --src [--src , ...] [--askVaultPassword] [--vaultPasswordFile ] --action [--workingFolder ] [--loggingConf ] ' + echo 'hadeploy --src [--src , ...] [--askVaultPassword] [--vaultPasswordFile ] [--workingFolder ] [--loggingConf ] --action ' echo ' or' echo 'hadeploy --version' } diff --git a/docsite/docs/gettingstarted.md b/docsite/docs/gettingstarted.md index 2b4a484..029e2ea 100644 --- a/docsite/docs/gettingstarted.md +++ b/docsite/docs/gettingstarted.md @@ -85,8 +85,8 @@ In such case, the infrastructure part of the file will look like: ```yaml # ------------------------------------- Infrastructure part -ansible_inventory_files: -- ".../some-ansible-build/inventory" +ansible_inventories: +- file: ".../some-ansible-build/inventory" host_groups: - name: control_nodes @@ -443,3 +443,9 @@ Note in this case, the value will be overwritten by the one provided in app.yml. The general rule is than variable definitions are evaluated/overridden in order of which they appears. So the order of --var and --src on the command line is significant. +Some other option of the HADeploy command are described in other chapters: + +* `--workingFolder`: Refer to [`Under_the_hood`](./more/under_the_hood) chapter. + +* `--askVaultPassword` and `--vaultPasswordFile`: Refer to [encrypted values](../../more/encrypted_values) + diff --git a/docsite/docs/more/encrypted_values.md b/docsite/docs/more/encrypted_values.md new file mode 100644 index 0000000..d9806d9 --- /dev/null +++ b/docsite/docs/more/encrypted_values.md @@ -0,0 +1,118 @@ +# Encrypted values + +Obviously, some value needs to be hidden. This is the case for the ranger admin password. + +HADeploy will allow such value to be encrypted. This can be achived by provided the values as in the following sample: + +```yaml +ranger_relay: + host: en1 + ranger_url: https://ranger.mycluster.mycompany.com:6182 + ranger_username: admin + ranger_password: | + $ANSIBLE_VAULT;1.1;AES256 + 34396662613462623565323936616330623661623065343033646136643635653430636238613962 + 3537343131346462343138343064313937646366363435340a633532366162623838376436366362 + 61393033343932303636653066336130616132383463373934396265306364363562613565613165 + 6163613739303430650a356136353865623534643237646166393230613933396166663963633538 + 3664 + ca_bundle_local_file: cert/ranger_mycluster_cert.pem + ca_bundle_relay_file: /etc/security/certs/ranger_mycluster_cert.pem +``` + +NB: On this version of HADeploy, only the `ranger_password` attribut of [`ranger_relay`](../plugins_reference/ranger/ranger_relay) support this feature. + +## Encrypting a value + +HADeploy encryption rely on the Ansible Vault capability. So, the encryption will be performed using `ansible-vault` commmand. + +Here is a simple approach to achieve this: + +First, create a temporary file containing only the password (Here, the password is `admin`): + +```bash +echo -n admin >/tmp/data.txt +``` + +It is important to ensure there is no leading or trailing control character, or white space: + + +```bash +hexdump -C /tmp/data.txt +00000000 61 64 6d 69 6e |admin| +00000005 +``` + +Then, you can encrypt it, using the following command: + +```bash +ansible-vault encrypt {"failed": true, "msg": "Unexpected templating type error occurred on ({{ rangerPassword }}): Non-hexadecimal digit found"} +``` + +And, of course, don't forget to cleanup the file which contains the password in clear text. + +``` +rm /tmp/data.txt +``` + +NB: The encrypted value is directly provided to Ansible, which will decrypt it in memory, at run time. In other word, HADeploy itself does not perform decryption. So, there is no risk to have a clear password in some intermediate file. + +## Launching HADeploy with encrypted values + +> Do not mistake this feature with the vault password you may need to provide when accessing an Ansible inventory ([see here](../plugins_reference/ansible_inventories/ansible_inventories)). +There is no relationshipt between these two passwords. They act at different level. + +If you launch HADeploy on file containing encrypted value, you will need to provide a password. Otherwise you will have an error like the following: + +```bash +The offending line appears to be: + + vars: + rangerPassword: !vault | + ^ here +``` + +First approach is to enter this password on each launch. For this, simply add the option `--askVaultPassword` on the command line. + +```bash +hadeploy --src infra/mycluster.yml --src app.yml --askVaultPassword --action DEPLOY +.... +Vault password: +``` + +Another approach is to provide this password in a file. The password must be a string stored as a single line of the file. + +Then use the option `--vaultPasswordFile` to provide the path on this file: + +```bash +hadeploy --src infra/mycluster.yml --src app.yml --vaultPasswordFile infra/vault_password.txt --action DEPLOY +``` + +Ensure permissions on the file are such that no one else can access your key and do not add this file to source control. + + + + diff --git a/docsite/docs/more/release_notes.md b/docsite/docs/more/release_notes.md index 5030606..be4c3ff 100644 --- a/docsite/docs/more/release_notes.md +++ b/docsite/docs/more/release_notes.md @@ -9,7 +9,7 @@ - `kafka_relay.tools_folder` default value is modified to be in `/tmp/hadeploy_{{ansible_ssh_user}}` - Local file definition now works with ~/xxx - `hdfs_relay.user` default value is now `hdfs` only if `ssh_user`is root. Otherwise, is `ssh_user` -- Added global `exit_on_fail` flag. +- Added global [`exit_on_fail`](../plugins_reference/inventory/exit_on_fail) flag. - Added some retry on user/groups creation/removal - Added `become_user/become_method` on `hbase_relay` - Added `become_user/become_method` on `kafka_relay` @@ -18,6 +18,13 @@ - Added `hdfs_relay.local_keytab_path` - Added `source_host_credential.local_keytab_path` - Added `groups` in `host_groups` +- Previous version was unable to fetch an existing Ansible inventory when it contains some encrypted file(s). +The Description of Ansible inventory has been modified (now `ansible_inventories`) to include a password file or user password request. +- Added a method to encrypt password in ranger_relay.admin_password. + +#### DEPRECATION + +`ansible_inventory_files` has been marked deprecated. Replaced by `ansible_inventories`. #### INCOMPATIBILITY diff --git a/docsite/docs/more/under_the_hood.md b/docsite/docs/more/under_the_hood.md index a6e0d35..95b7354 100644 --- a/docsite/docs/more/under_the_hood.md +++ b/docsite/docs/more/under_the_hood.md @@ -66,7 +66,7 @@ All theses are optional. A plugin can also host: Currently, HADeploy is provided with the following internal plugin: * `ìnventory` for target hosts management. -* `ansible_inventory` for direct integration of an Ansible inventory. +* `ansible_inventories` for direct integration of an Ansible inventories. * `users` for management of local users and groups. * `files` for base file management. * `hdfs` which extends the previous one for HDFS accesses. diff --git a/docsite/docs/plugins_reference/ansible_inventories/ansible_inventories.md b/docsite/docs/plugins_reference/ansible_inventories/ansible_inventories.md new file mode 100644 index 0000000..3afb4d9 --- /dev/null +++ b/docsite/docs/plugins_reference/ansible_inventories/ansible_inventories.md @@ -0,0 +1,46 @@ +# ansible_inventories + +## Synopsis + +Provide a list of Ansible Inventory. + +These inventories will be parsed, eventually merged with locally defined [`hosts`](../inventory/hosts) and [`host_groups`](../inventory/host_groups) and will allow target Ansible inventory file generation. + +Also, some paramters may be modified by using [`host_overrides`](../inventory/host_overrides.md) and [`host_group_overrides`](../inventory/host_group_overrides.md). + +## Attributes + +Each item of the list has the following attributes: + +Name | req? | Description +--- | --- | --- +file|yes|The Ansible inventory file path. If this path is not absolute, it will be relative to the HADeploy embedding file location. +vault_password_file|no|If this inventory host one or several encrypted files, one must provide a password for decryption. One method is to provide this password in a file.
The password must be a string stored as a single line of the file.
If this path is not absolute, it will be relative to the HADeploy embedding file location.
It can also be stored in the home folder of the sshd_user by using the ~/... notation.
In all cases, ensure permissions on the file are such that no one else can access your key and do not add your this file to source control. +ask_vault_password|no|Boolean. Another method to provide this password is to set this switch on. In this case, the user will be prompted to enter the password on each run. +name|no|Allow to provide a name to this inventory.
Useful if this list contains several entries which require a password. This name will be inserted in the prompt for the user. + + +## Example + +```yaml +# This simplest case, with a single inventory +ansible_inventories: +- file: ".../some-ansible-folder/inventory" + + +# Build our own inventory from two Ansible inventories. And request user password with decorated prompt +ansible_inventories: +- name: "inv1" + files: ".../some-ansible-folder/inventory" + ask_vault_password: yes +- name: "inv2" + file: ".../another-ansible-folder/inventory" + ask_vault_password: yes + +``` +## Inventory merging + +If a host with same name is defined both in [`hosts`](../inventory/hosts) and in an Ansible inventory, the one from the [`hosts`](../inventory/hosts) list will take precedence. +This is same for the [`host_groups`](../inventory/host_groups). + +Note also a [`host_groups`](../inventory/host_groups) can refer to a host in Ansible inventory. diff --git a/docsite/docs/plugins_reference/ansible_inventory/ansible_inventory_files.md b/docsite/docs/plugins_reference/ansible_inventory/ansible_inventory_files.md index a221ddb..a9d38d4 100644 --- a/docsite/docs/plugins_reference/ansible_inventory/ansible_inventory_files.md +++ b/docsite/docs/plugins_reference/ansible_inventory/ansible_inventory_files.md @@ -1,7 +1,11 @@ -# ansible_inventory_files +# ansible_inventory_files -## Synopsis +> **This feature is DEPRECATED. Please use [ansible_inventories](../ansible_inventories/ansible_inventories) instead.** + +> It will be removed in a next release. + +## Synopsis Define a list of Ansible Inventory files. These files will be parsed, eventually merged with locally defined hosts and host_groups and will allow target Ansible inventory file generation @@ -28,3 +32,4 @@ If a host with same name is defined both in [`hosts`](../inventory/hosts) and in This is same for the [`host_groups`](../inventory/host_groups). Note also a [`host_groups`](../inventory/host_groups) can refer to a host in Ansible inventory. + \ No newline at end of file diff --git a/docsite/docs/plugins_reference/ranger/ranger_relay.md b/docsite/docs/plugins_reference/ranger/ranger_relay.md index e927456..cbd51e6 100644 --- a/docsite/docs/plugins_reference/ranger/ranger_relay.md +++ b/docsite/docs/plugins_reference/ranger/ranger_relay.md @@ -14,7 +14,7 @@ Name | req? | Description host|yes|From which host these commands will be launched. Typically, an edge node in the cluster. But may be a host outside the cluster. ranger_url|yes|The Ranger base URL to access Ranger API. Same host:port as the Ranger Admin GUI. Typically:
`http://myranger.server.com:6080`
or
`https://myranger.server.com:6182` ranger_username|yes|The user name to log on the Ranger Admin. Must have enough rights to manage policies -ranger_password|yes|The password associated with the admin_username +ranger_password|yes|The password associated with the admin_username. May be encrypted. Refer to [encrypted values](../../more/encrypted_values) validate_certs|no|Useful if Ranger Admin connection is using SSL. If no, SSL certificates will not be validated. This should only be used on personally controlled sites using self-signed certificates.
Default: `yes` ca_bundle_relay_file|no|Useful if Ranger Admin connection is using SSL. Allow to specify a CA_BUNDLE file, a file that contains root and intermediate certificates to validate the Ranger Admin certificate.
In its simplest case, it could be a file containing the server certificate in .pem format. This file will be looked up on the relay host system, on which this module will be executed. ca_bundle_local_file|no|Same as above, except this file will be looked up locally, relative to the main file. It will be copied on the relay host at the location defined by ca_bundle_relay_file @@ -33,16 +33,36 @@ ranger_relay: ranger_username: admin ranger_password: admin ``` -A more secure configuration, with https and certificate validation: +A more secure configuration, with https, certificate validation and encrypted password. ```yaml ranger_relay: host: en1 ranger_url: https://ranger.mycluster.mycompany.com:6182 ranger_username: admin - ranger_password: aYhGhP6= + ranger_password: | + $ANSIBLE_VAULT;1.1;AES256 + 34396662613462623565323936616330623661623065343033646136643635653430636238613962 + 3537343131346462343138343064313937646366363435340a633532366162623838376436366362 + 61393033343932303636653066336130616132383463373934396265306364363562613565613165 + 6163613739303430650a356136353865623534643237646166393230613933396166663963633538 + 3664 ca_bundle_local_file: cert/ranger_mycluster_cert.pem ca_bundle_relay_file: /etc/security/certs/ranger_mycluster_cert.pem ``` + +When using such encryption feature, you will need to provide a password when launching HADeploy. Otherwise you will have an error like the following: + +```bash +The offending line appears to be: + + vars: + rangerPassword: !vault | + ^ here +``` + + +More detail on how to encrypt a value and providing a password on executuion at [encrypted values](../../more/encrypted_values) + ## Simple CA_BUNDLE In its simplest case, a CA_BUNDLE can be simply the certificate of the Ranger server, in PEM format. diff --git a/docsite/mkdocs.yml b/docsite/mkdocs.yml index 1b7e2c9..54c0874 100644 --- a/docsite/mkdocs.yml +++ b/docsite/mkdocs.yml @@ -10,6 +10,8 @@ pages: - vars: plugins_reference/core/vars.md - Ansible_inventory: - ansible_inventory_files: plugins_reference/ansible_inventory/ansible_inventory_files.md + - Ansible_inventories: + - ansible_inventories: plugins_reference/ansible_inventories/ansible_inventories.md - Files: - files: plugins_reference/files/files.md - folders: plugins_reference/files/folders.md @@ -52,6 +54,7 @@ pages: - "Execution order": more/execution_order.md - "Secured cluster": more/secured_cluster.md - "YAML tricks": more/yaml_tricks.md + - "Encrypted values": more/encrypted_values.md - "Release notes": more/release_notes.md #theme: readthedocs