Merge Release 2.2.1

Merge Release 2.2.1

cli/.isort.cfg → .isort.cfg

@@ -1,7 +1,7 @@
 [settings]
 line_length=120
 known_future_library=future
-known_third_party=boto3,botocore,awscli,tabulate,argparse,configparser,pytest
+known_third_party=boto3,botocore,awscli,tabulate,argparse,configparser,pytest,pytest,pytest-datadir,pytest-html,pytest-rerunfailures,pytest-xdist,argparse,retrying,junitparser,Jinja2
 # 3 - Vertical Hanging Indent
 # from third_party import (
 #     lib1,

CHANGELOG.rst

@@ -2,15 +2,65 @@
 CHANGELOG
 =========
 
+2.2.1
+=====
+
+**ENHANCEMENTS**
+
+* Add support for FSx Lustre in Centos 7. In case of custom AMI, FSx Lustre is
+  only supported with Centos 7.5 and Centos 7.6.
+* Check AWS EC2 instance account limits before starting cluster creation
+* Allow users to force job deletion with ``SGE`` scheduler
+
+**CHANGES**
+
+* Set default value to ``compute`` for ``placement_group`` option
+* ``pcluster ssh``: use private IP when the public one is not available
+* ``pcluster ssh``: now works also when stack is not completed as long as the master IP is available
+* Remove unused dependency on ``awscli`` from ParallelCluster package
+
+**BUG FIXES**
+
+* ``awsbsub``: fix file upload with absolute path
+* ``pcluster ssh``: fix issue that was preventing the command from working correctly when stack status is
+  ``UPDATE_ROLLBACK_COMPLETE``
+* Fix block device conversion to correctly attach EBS nvme volumes
+* Wait for Torque scheduler initialization before completing master node setup
+* ``pcluster version``: now works also when no ParallelCluster config is present
+* Improve ``nodewatcher`` daemon logic to detect if a SGE compute node has running jobs
+
+**DOCS**
+
+* Add documentation on how to use FSx Lustre
+* Add tutorial for encrypted EBS with a Custom KMS Key
+* Add ``ebs_kms_key_id`` to Configuration section
+
+**TESTING**
+
+* Define a new framework to write and run ParallelCluster integration tests
+* Improve scaling integration tests to detect over-scaling
+* Implement integration tests for awsbatch scheduler
+* Implement integration tests for FSx Lustre file system
+
 2.1.1
 =====
-* China regions `cn-north-1` and `cn-northwest-1`
+* Add China regions `cn-north-1` and `cn-northwest-1`
 
 2.1.0
 =====
-* RAID support
-* EFS support
-* AWS Batch Multinode Parallel Support
+* Add configuration for RAID 0 and 1 volumes
+* Add Elastic File System (EFS) support
+* Add AWS Batch Multinode Parallel jobs support
+* Add support for Stockholm region (`eu-north-1`)
+* Add `--env` and `--env-blacklist` options to the `awsbsub` command to export environment variables
+  in the job environment
+* Add `--input-file` option to the `awsbsub` command to stage-in input files from the client
+* Add new `PCLUSTER_JOB_S3_URL` variable to the job execution environment pointing to the S3 URL used
+  for job data stage-in/out
+* Add S3 URL for job data staging to the `awsbstat -d` output
+* Add `--working-dir` and `--parent-working-dir` options to the `awsbsub` command to specify
+  the working-directory or the parent working directory for the job
+* Add CPUs and Memory information to the `awsbhosts -d` command
 
 2.0.2
 =====
@@ -56,41 +106,41 @@ CHANGELOG
 1.5.2
 =====
 * feature:``cfncluster``: Added ClusterUser as a stack output. This makes it easier to get the username of the head node.
-* feature:``cfncluster``: Added `cfncluster ssh cluster_name`, this allows you to easily ssh into your clusters. It allows arbitrary command execution and extra ssh flags to be provided after the command. See https://aws-parallelcluster.readthedocs.io/en/latest/commands.html#ssh
-* change:``cfncluster``: Moved global cli flags to the command specific flags. For example `cfncluster --region us-east-1 create` now becomes `cfncluster create --region us-east-1`
+* feature:``cfncluster``: Added `cfncluster ssh cluster_name`, this allows you to easily ssh into your clusters.
+  It allows arbitrary command execution and extra ssh flags to be provided after the command.
+  See https://aws-parallelcluster.readthedocs.io/en/latest/commands.html#ssh
+* change:``cfncluster``: Moved global cli flags to the command specific flags.
+  For example `cfncluster --region us-east-1 create` now becomes `cfncluster create --region us-east-1`
 * bugfix:``cfncluster-cookbook``: Fix bug that prevented c5d/m5d instances from working
 * bugfix:``cfncluster-cookbook``: Set CPU as a consumable resource in slurm
 * bugfix:``cfncluster-node``: Fixed Slurm behavior to add CPU slots so multiple jobs can be scheduled on a single node
 
 1.5.1
 =====
 * change:``cfncluster``: Added "ec2:DescribeVolumes" permissions to
-CfnClusterInstancePolicy
+  CfnClusterInstancePolicy
 * change:``cfncluster``: Removed YAML CloudFormation template, it can be
-generated by the https://github.com/awslabs/aws-cfn-template-flip tool
+  generated by the https://github.com/awslabs/aws-cfn-template-flip tool
 
 * updates:``cfncluster``: Add support for eu-west-3 region
 
 * feature:``cfncluster-cookbook``: Added parameter to specify custom
-cfncluster-node package
+  cfncluster-node package
 
 * bugfix:``cfncluster``: Fix --template-url command line parameter
 * bugfix:``cfncluster-cookbook``: Poll on EBS Volume attachment status
-* bugfix:``cfncluster-cookbook``: Fixed SLURM cron job to publish pending
-metric
-* bugfix:``cfncluster-node``: Fixed Torque behaviour when scaling up from an
-empty cluster
+* bugfix:``cfncluster-cookbook``: Fixed SLURM cron job to publish pending metric
+* bugfix:``cfncluster-node``: Fixed Torque behaviour when scaling up from an empty cluster
 
 
 1.4.2
 =====
 * bugfix:``cfncluster``: Fix crash when base directory for config file
   does not exist
 * bugfix:``cfncluster``: Removed extraneous logging message at
-  cfncluster invocation, re-enabled logging in
-  ~/.cfncluster/cfncluster-cli.log
+  cfncluster invocation, re-enabled logging in ~/.cfncluster/cfncluster-cli.log
 * bugfix: ``cfncluster-node``: Fix scaling issues with CentOS 6 clusters caused
-by incompatible dependencies.
+  by incompatible dependencies.
 * updates:``ami``: Update all base AMIs to latest patch levels
 * updates:``cfncluster-cookbook``: Updated to cfncluster-cookbook-1.4.1
 
@@ -161,7 +211,7 @@ by incompatible dependencies.
 
 1.0.0
 =====
-Offiical release of the CfnCluster 1.x CLI, templates and AMIs. Available in all regions except BJS, with
+Official release of the CfnCluster 1.x CLI, templates and AMIs. Available in all regions except BJS, with
 support for Amazon Linux, CentOS 6 & 7 and Ubuntu 14.04 LTS. All AMIs are built via packer from the CfnCluster
 Cookbook project (https://github.com/aws/aws-parallelcluster-cookbook).
 

amis.txt

@@ -1,100 +1,100 @@
 # alinux
-ap-northeast-1: ami-0ac1d0c35dc3b2a97
-ap-northeast-2: ami-0dddb89f66485f828
-ap-northeast-3: ami-0dbb09bfada65298f
-ap-south-1: ami-0ed1bb8f2ab0edffb
-ap-southeast-1: ami-058bcd8377aba9bef
-ap-southeast-2: ami-012557ce9426ef1a0
-ca-central-1: ami-059dda8ee9af7c20c
-cn-north-1: ami-00237da8e056b0936
-cn-northwest-1: ami-0974fc483e449f5ee
-eu-central-1: ami-09cff6787920e967c
-eu-north-1: ami-086f4f382fb1119f0
-eu-west-1: ami-080a7d5c75253bd1b
-eu-west-2: ami-0b284f1028a743865
-eu-west-3: ami-099197e40d0c5de6e
-sa-east-1: ami-02e542e4935ff9647
-us-east-1: ami-0cd2dd3198972a68c
-us-east-2: ami-057e70f0fbb007ab6
-us-gov-east-1: ami-022f96b137a63c9ff
-us-gov-west-1: ami-fefb989f
-us-west-1: ami-09686d4090e35a702
-us-west-2: ami-0c588cdc9e91b0db3
+ap-northeast-1: ami-01e7e307e734daa9b
+ap-northeast-2: ami-096da9252851971b7
+ap-northeast-3: ami-0fde1cce0915721ec
+ap-south-1: ami-0856113449f34cfde
+ap-southeast-1: ami-019762c344b80574a
+ap-southeast-2: ami-09ddff457fb3815ea
+ca-central-1: ami-0ef3d34adf231688a
+cn-north-1: ami-053a5a11a4ac83842
+cn-northwest-1: ami-0c5a99564e44467e7
+eu-central-1: ami-01d7252afc45b0d8b
+eu-north-1: ami-02224ba7786413561
+eu-west-1: ami-0a32ae196621ce1cd
+eu-west-2: ami-0e8d810df84d61f8b
+eu-west-3: ami-09a1847f7683c6351
+sa-east-1: ami-03c4f6e67dcea925d
+us-east-1: ami-096b5898281e68ea3
+us-east-2: ami-0c8b41c511db3c17c
+us-gov-east-1: ami-08cff3f9ef830bdfb
+us-gov-west-1: ami-afa0c8ce
+us-west-1: ami-08e9806b160f9aa44
+us-west-2: ami-04eba5b9de0d94ec6
 # centos6
-ap-northeast-1: ami-003cfe6266cadd576
-ap-northeast-2: ami-032b3f2a4f1ac91a0
-ap-northeast-3: ami-0c4de7aece7b2db33
-ap-south-1: ami-0581583c3d7507d9e
-ap-southeast-1: ami-0d00309b80b772532
-ap-southeast-2: ami-00ecf7e455945e8bc
-ca-central-1: ami-0416d41ed6dbc0bd7
-eu-central-1: ami-0b1f52047bb2b7f83
-eu-north-1: ami-04fc976e8108996e6
-eu-west-1: ami-073f1f5db6cfdd3d1
-eu-west-2: ami-0ace56d1d9c1aa466
-eu-west-3: ami-0d377bccde07c887d
-sa-east-1: ami-0b1c8f6aad337d5b6
-us-east-1: ami-0919d912e0e33d247
-us-east-2: ami-0bbd43b2b8991cdae
-us-west-1: ami-0ac1cf1e68288fa36
-us-west-2: ami-08b28682da5721f5b
+ap-northeast-1: ami-09c0d7b7eba653962
+ap-northeast-2: ami-04bb0577f1425f61f
+ap-northeast-3: ami-092c70a47ecb4f730
+ap-south-1: ami-0b6b4a5c5952ba214
+ap-southeast-1: ami-0d8fee73833faed87
+ap-southeast-2: ami-03e0e45d59cf130b3
+ca-central-1: ami-0f5cabbce86422cc0
+eu-central-1: ami-0c47523135cb69662
+eu-north-1: ami-064af23ad3c8bac88
+eu-west-1: ami-048edb0ed06f91ef8
+eu-west-2: ami-098b95acdef72fd6c
+eu-west-3: ami-0e25f1a2ef5f8d237
+sa-east-1: ami-07bcda88df87d3db2
+us-east-1: ami-0fd0ae64eb509cc23
+us-east-2: ami-09c42f6a076482eeb
+us-west-1: ami-060bcf74fb6278986
+us-west-2: ami-009c08111f57a2d4c
 # centos7
-ap-northeast-1: ami-0c0049e4eeb0ef1ac
-ap-northeast-2: ami-00f8f46a043a04530
-ap-northeast-3: ami-0043e5e12872a00d4
-ap-south-1: ami-03dd063b05c3082f1
-ap-southeast-1: ami-00fcdc55bd29f691e
-ap-southeast-2: ami-0baa1787fd7a71950
-ca-central-1: ami-0c2a8b54dfd0c405f
-eu-central-1: ami-00b3f34240b6021dd
-eu-north-1: ami-07ac74edc9e96b343
-eu-west-1: ami-0879d97613ba8075a
-eu-west-2: ami-06aa0ca6f28c29fdb
-eu-west-3: ami-0b28dcaf2b4d00d87
-sa-east-1: ami-00a570d9536621660
-us-east-1: ami-09d092c1b054832df
-us-east-2: ami-0e0cda1cdec7fcecf
-us-west-1: ami-084c065e503e74449
-us-west-2: ami-070345cb145f2d2f0
+ap-northeast-1: ami-0307ad022fdd9bd50
+ap-northeast-2: ami-01610b2d147974908
+ap-northeast-3: ami-04dd95194c8990d40
+ap-south-1: ami-0d848603d9bf30b76
+ap-southeast-1: ami-084b71a47f7c857b7
+ap-southeast-2: ami-0b06de8c266942eff
+ca-central-1: ami-0a2b442fa56fe5db0
+eu-central-1: ami-089c27327ed56f33c
+eu-north-1: ami-016fe6e29bff94e38
+eu-west-1: ami-00e9df6764697605f
+eu-west-2: ami-00884b56887c9585a
+eu-west-3: ami-0252465d6dff43cfb
+sa-east-1: ami-02f4e6da1e0de2347
+us-east-1: ami-08b43bfe7a0d16eeb
+us-east-2: ami-0c71e12e95cc937ef
+us-west-1: ami-0f925f166f349eaa4
+us-west-2: ami-05f6a646767bf0c87
 # ubuntu1404
-ap-northeast-1: ami-053a84278e34b7a59
-ap-northeast-2: ami-03d314c5ac10beb53
-ap-northeast-3: ami-070f9069b20a797ec
-ap-south-1: ami-0641d7ca885530589
-ap-southeast-1: ami-0e3de99412375e882
-ap-southeast-2: ami-09eae4580e8fc835a
-ca-central-1: ami-08aeb7a57f73b58ab
-cn-north-1: ami-00f2cae5406fb3fce
-eu-central-1: ami-0b24a435216670b4a
-eu-north-1: ami-0921b515f8ed512c3
-eu-west-1: ami-076fbdec21cd5c940
-eu-west-2: ami-0fb01b5b56bc27509
-eu-west-3: ami-0ea8b79c622d0a4a2
-sa-east-1: ami-00cf3910c959e9fd5
-us-east-1: ami-095016c5ff0ab7ae6
-us-east-2: ami-02a21f90af8a453f0
-us-gov-east-1: ami-09274e2587c4154f2
-us-gov-west-1: ami-6cf0930d
-us-west-1: ami-099ebeb456dbf0646
-us-west-2: ami-05ddc7ec0edb9069f
+ap-northeast-1: ami-0141f9239dd88f7eb
+ap-northeast-2: ami-069d8e1e9b74ee48d
+ap-northeast-3: ami-07853be3b108fb14c
+ap-south-1: ami-02744b942f4094210
+ap-southeast-1: ami-070fef5acc9fd3de2
+ap-southeast-2: ami-097c066ff342e12fa
+ca-central-1: ami-075be3409c2c2738d
+cn-north-1: ami-0e90250aa31ac4a12
+eu-central-1: ami-093d2529752e5e2f2
+eu-north-1: ami-01ea61e8b26299de1
+eu-west-1: ami-0d1cb938a6d6b693c
+eu-west-2: ami-0487ce94890b5869b
+eu-west-3: ami-01a321e226dbf081a
+sa-east-1: ami-02366044e8d7372fa
+us-east-1: ami-00f8fb8ed06007c1f
+us-east-2: ami-0e23f662335e169a6
+us-gov-east-1: ami-0a34c5de0403642d2
+us-gov-west-1: ami-e4a3cb85
+us-west-1: ami-021201edbaab31f29
+us-west-2: ami-085bb9cfafd2e3b3a
 # ubuntu1604
-ap-northeast-1: ami-026669cfcef23b3de
-ap-northeast-2: ami-048214d1413ed8462
-ap-northeast-3: ami-038900edb73cb9496
-ap-south-1: ami-0cc424ec58256ea88
-ap-southeast-1: ami-059ba95190db36590
-ap-southeast-2: ami-04df2433ab61d3f37
-ca-central-1: ami-039a1b0ada060b5ce
-cn-north-1: ami-072046713a0458796
-eu-central-1: ami-0d816068d1164f4d2
-eu-north-1: ami-046c32486a9abf742
-eu-west-1: ami-0f641e63ebaf647b1
-eu-west-2: ami-067c1c0157477c166
-eu-west-3: ami-0102caf8c6ec0768a
-sa-east-1: ami-0ef5c70aec338bcfb
-us-east-1: ami-098c8e582ca818cff
-us-east-2: ami-055279b0b09d12a71
-us-gov-east-1: ami-0b664ce8c427b77eb
-us-gov-west-1: ami-5ef6953f
-us-west-1: ami-0db5e85c0b1ce2c20
-us-west-2: ami-02393fa61ac61547a
+ap-northeast-1: ami-00bacec2848062b6b
+ap-northeast-2: ami-0a6edfea96ed6c9db
+ap-northeast-3: ami-09eeb02948489e793
+ap-south-1: ami-0dcc47340ec3a0e45
+ap-southeast-1: ami-0771cf73e73259040
+ap-southeast-2: ami-0252c6d83fa35183c
+ca-central-1: ami-090b3ee62a0b5412b
+cn-north-1: ami-0b89a3058ea57b76f
+eu-central-1: ami-0c7ccc7ec89bd0d75
+eu-north-1: ami-0675d81a659cb530e
+eu-west-1: ami-010af7d7788dac778
+eu-west-2: ami-01290e9a5298ae134
+eu-west-3: ami-0996ed0efb2b8cc4a
+sa-east-1: ami-0a3ef670136cc81ee
+us-east-1: ami-05f79ab77fc13e20a
+us-east-2: ami-08a675767416c627a
+us-gov-east-1: ami-08b15aaa28124fac1
+us-gov-west-1: ami-eba3cb8a
+us-west-1: ami-059b8cd1b0041071a
+us-west-2: ami-034bffb8da06d6951

cli/.flake8

@@ -9,13 +9,18 @@ ignore =
 # E402 module level import not at top of file
 # D101 Missing docstring in public class
 # D102 Missing docstring in public method
+# D205 1 blank line required between summary line and description
+# D400 First line should end with a period
+# D401 First line should be in imperative mood
 per-file-ignores =
     pcluster/*.py: D103
-    pcluster/config_sanity.py: E402
-    pcluster/easyconfig.py: E402
-    pcluster/cfnconfig.py: E402
-    tests/pcluster/pcluster-unittest.py: D101, D102
+    pcluster/config_sanity.py: E402, D103
+    pcluster/easyconfig.py: E402, D103
+    pcluster/cfnconfig.py: E402, D103
+    tests/pcluster/pcluster-unittest.py: D101, D102, D103
     tests/awsbatch/test_*.py: D101, D102
+    ../tests/integration-tests/tests/*: D103
+    ../tests/integration-tests/*: D205, D400, D401
 exclude =
     .tox,
     .git,

cli/awsbatch/awsbsub.py

@@ -214,7 +214,7 @@ def _upload_and_get_command(boto3_factory, args, job_s3_folder, job_name, config
     # upload input files, if there
     if args.input_file:
         for file in args.input_file:
-            s3_uploader.put_file(file, file)
+            s3_uploader.put_file(file, os.path.basename(file))
 
     # upload command, if needed
     if args.command_file or not sys.stdin.isatty() or args.env: