Add --timeout-millis option to export-pg-from-queries (#77)

* Add --timeout-millis option to export-pg-from-queries, and update docs * Update list of clone instance types * Update docs
awslabs · Sep 8, 2020 · 8e57ea5 · 8e57ea5
1 parent 5fb4f7f
commit 8e57ea5
Show file tree

Hide file tree

Showing 13 changed files with 472 additions and 54 deletions.
diff --git a/neptune-export/docs/add-clone.md b/neptune-export/docs/add-clone.md
@@ -0,0 +1,52 @@
+    NAME
+            neptune-export.sh add-clone - Clone an Amazon Neptune database cluster
+
+    SYNOPSIS
+            neptune-export.sh add-clone [ --clone-cluster-id <targetClusterId> ]
+                    [ --clone-cluster-instance-type <cloneClusterInstanceType> ]
+                    [ --clone-cluster-replica-count <replicaCount> ]
+                    --source-cluster-id <sourceClusterId>
+
+    OPTIONS
+            --clone-cluster-id <targetClusterId>
+                Cluster ID of the cloned Amazon Neptune database cluster
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster-instance-type <cloneClusterInstanceType>
+                Instance type for cloned cluster (by default neptune-export will
+                use the same instance type as the source cluster)
+
+                This options value is restricted to the following set of values:
+                    db.r4.large
+                    db.r4.xlarge
+                    db.r4.2xlarge
+                    db.r4.4xlarge
+                    db.r4.8xlarge
+                    db.r5.large
+                    db.r5.xlarge
+                    db.r5.2xlarge
+                    db.r5.4xlarge
+                    db.r5.8xlarge
+                    db.r5.12xlarge
+                    db.t3.medium
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster-replica-count <replicaCount>
+                Number of read replicas to add to the cloned cluster (default, 0)
+
+                This option may occur a maximum of 1 times
+
+
+                This options value must fall in the following range: 0 <= value <= 15
+
+
+            --source-cluster-id <sourceClusterId>
+                Cluster ID of the source Amazon Neptune database cluster
+
+                This option may occur a maximum of 1 times
+
+
diff --git a/neptune-export/docs/create-pg-config.md b/neptune-export/docs/create-pg-config.md
@@ -5,17 +5,21 @@
     SYNOPSIS
             neptune-export.sh create-pg-config
                     [ --alb-endpoint <applicationLoadBalancerEndpoint> ]
-                    [ {-b | --batch-size} <batchSize> ]
+                    [ {-b | --batch-size} <batchSize> ] [ --clone-cluster ]
+                    [ --clone-cluster-instance-type <cloneClusterInstanceType> ]
+                    [ --clone-cluster-replica-count <replicaCount> ]
+                    [ --cluster-id <clusterId> ]
                     [ {-cn | --concurrency} <concurrency> ]
-                    {-d | --dir} <directory> {-e | --endpoint} <endpoint>...
-                    [ {-el | --edge-label} <edgeLabels>... ]
+                    {-d | --dir} <directory> [ {-e | --endpoint} <endpoint>... ]
+                    [ {-el | --edge-label} <edgeLabels>... ] [ --format <format> ]
                     [ --lb-port <loadBalancerPort> ] [ --log-level <log level> ]
                     [ --max-content-length <maxContentLength> ]
                     [ {-nl | --node-label} <nodeLabels>... ]
                     [ --nlb-endpoint <networkLoadBalancerEndpoint> ]
-                    [ {-p | --port} <port> ] [ {-s | --scope} <scope> ]
-                    [ --sample ] [ --sample-size <sampleSize> ]
-                    [ --serializer <serializer> ] [ {-t | --tag} <tag> ]
+                    [ {-o | --output} <output> ] [ {-p | --port} <port> ]
+                    [ --region <region> ] [ {-s | --scope} <scope> ] [ --sample ]
+                    [ --sample-size <sampleSize> ] [ --serializer <serializer> ]
+                    [ --stream-name <streamName> ] [ {-t | --tag} <tag> ]
                     [ --tokens-only <tokensOnly> ] [ --use-iam-auth ] [ --use-ssl ]
 
     OPTIONS
@@ -38,6 +42,61 @@
                 This option may occur a maximum of 1 times
 
 
+            --clone-cluster
+                Clone Neptune cluster
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster-instance-type <cloneClusterInstanceType>
+                Instance type for cloned cluster (by default neptune-export will
+                use the same instance type as the source cluster)
+
+                This options value is restricted to the following set of values:
+                    db.r4.large
+                    db.r4.xlarge
+                    db.r4.2xlarge
+                    db.r4.4xlarge
+                    db.r4.8xlarge
+                    db.r5.large
+                    db.r5.xlarge
+                    db.r5.2xlarge
+                    db.r5.4xlarge
+                    db.r5.8xlarge
+                    db.r5.12xlarge
+                    db.r5.16xlarge
+                    db.r5.24xlarge
+                    db.m5.large
+                    db.m5.xlarge
+                    db.m5.2xlarge
+                    db.m5.3xlarge
+                    db.m5.8xlarge
+                    db.m5.12xlarge
+                    db.m5.16xlarge
+                    db.m5.24xlarge
+                    db.t3.medium
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster-replica-count <replicaCount>
+                Number of read replicas to add to the cloned cluster (default, 0)
+
+                This option may occur a maximum of 1 times
+
+
+                This options value must fall in the following range: 0 <= value <= 15
+
+
+            --cluster-id <clusterId>
+                ID of an Amazon Neptune cluster. If you specify a cluster ID,
+                neptune-export will use all of the instance endpoints in the
+                cluster in addition to any endpoints you have specified using the
+                -e and --endpoint options.
+
+                This option may occur a maximum of 1 times
+
+
             -cn <concurrency>, --concurrency <concurrency>
                 Concurrency (optional)
 
@@ -62,6 +121,18 @@
                 Labels of edges to be included in config (optional, default all
                 labels)
 
+            --format <format>
+                Output format (optional, default 'csv')
+
+                This options value is restricted to the following set of values:
+                    csv
+                    csvNoHeaders
+                    json
+                    neptuneStreamsJson
+
+                This option may occur a maximum of 1 times
+
+
             --lb-port <loadBalancerPort>
                 Load balancer port (optional, default 80)
 
@@ -107,6 +178,17 @@
                 one option may be specified
 
 
+            -o <output>, --output <output>
+                Output target (optional, default 'file')
+
+                This options value is restricted to the following set of values:
+                    files
+                    stdout
+                    stream
+
+                This option may occur a maximum of 1 times
+
+
             -p <port>, --port <port>
                 Neptune port (optional, default 8182)
 
@@ -117,6 +199,12 @@
                 following port ranges: 1-1023, 1024-49151
 
 
+            --region <region>
+                AWS Region in which your Amazon Kinesis Data Stream is located
+
+                This option may occur a maximum of 1 times
+
+
             -s <scope>, --scope <scope>
                 Scope (optional, default 'all')
 
@@ -152,6 +240,12 @@
                 This option may occur a maximum of 1 times
 
 
+            --stream-name <streamName>
+                Name of an Amazon Kinesis Data Stream
+
+                This option may occur a maximum of 1 times
+
+
             -t <tag>, --tag <tag>
                 Directory prefix (optional)
 
@@ -172,8 +266,7 @@
 
             --use-iam-auth
                 Use IAM database authentication to authenticate to Neptune
-                (remember to set SERVICE_REGION environment variable, and, if using
-                a load balancer, set the --host-header option as well)
+                (remember to set SERVICE_REGION environment variable)
 
                 This option may occur a maximum of 1 times
 

diff --git a/neptune-export/docs/export-pg-from-config.md b/neptune-export/docs/export-pg-from-config.md
@@ -5,10 +5,15 @@
     SYNOPSIS
             neptune-export.sh export-pg-from-config
                     [ --alb-endpoint <applicationLoadBalancerEndpoint> ]
+                    [ --approx-edge-count <approxEdgeCount> ]
+                    [ --approx-node-count <approxNodeCount> ]
                     [ {-b | --batch-size} <batchSize> ]
-                    {-c | --config-file} <configFile>
+                    {-c | --config-file} <configFile> [ --clone-cluster ]
+                    [ --clone-cluster-instance-type <cloneClusterInstanceType> ]
+                    [ --clone-cluster-replica-count <replicaCount> ]
+                    [ --cluster-id <clusterId> ]
                     [ {-cn | --concurrency} <concurrency> ]
-                    {-d | --dir} <directory> {-e | --endpoint} <endpoint>...
+                    {-d | --dir} <directory> [ {-e | --endpoint} <endpoint>... ]
                     [ {-el | --edge-label} <edgeLabels>... ]
                     [ --exclude-type-definitions ] [ --format <format> ]
                     [ --lb-port <loadBalancerPort> ] [ --limit <limit> ]
@@ -36,6 +41,18 @@
                 one option may be specified
 
 
+            --approx-edge-count <approxEdgeCount>
+                Approximate number of edges in graph
+
+                This option may occur a maximum of 1 times
+
+
+            --approx-node-count <approxNodeCount>
+                Approximate number of nodes in graph
+
+                This option may occur a maximum of 1 times
+
+
             -b <batchSize>, --batch-size <batchSize>
                 Batch size (optional, default 64). Reduce this number if your
                 queries trigger CorruptedFrameExceptions.
@@ -44,14 +61,64 @@
 
 
             -c <configFile>, --config-file <configFile>
-                Path to JSON config file
+                Path to JSON config file (file path, or 'https' or 's3' URI)
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster
+                Clone Neptune cluster
+
+                This option may occur a maximum of 1 times
+
+
+            --clone-cluster-instance-type <cloneClusterInstanceType>
+                Instance type for cloned cluster (by default neptune-export will
+                use the same instance type as the source cluster)
+
+                This options value is restricted to the following set of values:
+                    db.r4.large
+                    db.r4.xlarge
+                    db.r4.2xlarge
+                    db.r4.4xlarge
+                    db.r4.8xlarge
+                    db.r5.large
+                    db.r5.xlarge
+                    db.r5.2xlarge
+                    db.r5.4xlarge
+                    db.r5.8xlarge
+                    db.r5.12xlarge
+                    db.r5.16xlarge
+                    db.r5.24xlarge
+                    db.m5.large
+                    db.m5.xlarge
+                    db.m5.2xlarge
+                    db.m5.3xlarge
+                    db.m5.8xlarge
+                    db.m5.12xlarge
+                    db.m5.16xlarge
+                    db.m5.24xlarge
+                    db.t3.medium
 
                 This option may occur a maximum of 1 times
 
 
-                This options value must be a path to a file. The provided path must
-                exist on the file system. The provided path must be readable and
-                writable.
+            --clone-cluster-replica-count <replicaCount>
+                Number of read replicas to add to the cloned cluster (default, 0)
+
+                This option may occur a maximum of 1 times
+
+
+                This options value must fall in the following range: 0 <= value <= 15
+
+
+            --cluster-id <clusterId>
+                ID of an Amazon Neptune cluster. If you specify a cluster ID,
+                neptune-export will use all of the instance endpoints in the
+                cluster in addition to any endpoints you have specified using the
+                -e and --endpoint options.
+
+                This option may occur a maximum of 1 times
 
 
             -cn <concurrency>, --concurrency <concurrency>
@@ -235,8 +302,7 @@
 
             --use-iam-auth
                 Use IAM database authentication to authenticate to Neptune
-                (remember to set SERVICE_REGION environment variable, and, if using
-                a load balancer, set the --host-header option as well)
+                (remember to set SERVICE_REGION environment variable)
 
                 This option may occur a maximum of 1 times