From 8b7a41e376cca6a4ceb820744f5bf8c8cfc8a171 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 3 Mar 2022 10:06:27 -0800
Subject: [PATCH 01/48] feat(accountability): synced accountability code with
 nisa. enabled accountability sciflo step

- changed: synced accountability.py with NISAR. updated NISAR references
- changed: enabled SCIFLO post-processing steps for accountability.
- changed: synced postprocess_functions.py with NISAR.
- changed: enable commit of .idea files per JetBrains recommendation
---
 .gitignore                                    |   2 +-
 opera_chimera/accountability.py               | 357 +++++++++---------
 .../configs/pge_configs/PGE_L3_HLS.yaml       |   2 +-
 opera_chimera/postprocess_functions.py        |  44 +--
 opera_chimera/wf_xml/L3_HLS.sf.xml            |   8 +-
 5 files changed, 199 insertions(+), 214 deletions(-)

diff --git a/.gitignore b/.gitignore
index e04d8a811..66f7c88cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -201,7 +201,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-.idea/
+#.idea/
 
 ######################################################################
 # JetBrains IDEs
diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index bed236fa3..578d75658 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -1,199 +1,208 @@
-import traceback
+import backoff
+import re
+import json
 import os
 from datetime import datetime
 
-from util.common_util import convert_datetime
-
 from chimera.commons.accountability import Accountability
 
 from opera_chimera.constants.opera_chimera_const import (
     OperaChimeraConstants as oc_const,
 )
 
+from util.conf_util import SettingsConf
 from chimera.logger import logger
 
-from commons.es_connection import get_grq_es
-# from pass_accountability.es_connection import get_pass_accountability_connection
-# from observation_accountability.es_connection import get_observation_accountability_connection
+from job_accountability.es_connection import get_job_accountability_connection
+
+grq_es = get_job_accountability_connection(logger)
 
-accountability_es = get_grq_es(logger)
-# accountability_es = get_pass_accountability_connection(logger)
-# obs_es = get_observation_accountability_connection(logger)
 
 PGE_STEP_DICT = {
-#    "L0A": "L0A_L_RRST_PP"
+    # "L0A": "L0A_L_RRST_PP"
 }
 
-INDECES = {
-    "": None,
-#    "pass": oc_const.PASS_ACCOUNTABILITY_INDEX,
-#    "observation": oc_const.OBSERVATION_ACCOUNTABILITY_INDEX,
-#    "track_frame": oc_const.TRACK_FRAME_ACCOUNTABILITY_INDEX
-}
+
+def get_dataset(key, datasets_cfg):
+    dataset = None
+    for ds in datasets_cfg["datasets"]:
+        dataset_regex = re.compile(ds["match_pattern"])
+        file_name = os.path.basename(key)
+        match = dataset_regex.search("/{}".format(file_name))
+        if match:
+            group_dict = match.groupdict()
+            ipath = ds["ipath"].format(**group_dict)
+            dataset = ipath.split("hysds::data/", 1)[1]
+            break
+
+    return dataset
+
+
+def get_dataset_type(file):
+    cfg = SettingsConf().cfg
+    data_name = None
+    for type, type_cfg in list(cfg["PRODUCT_TYPES"].items()):
+        matched = type_cfg["Pattern"].match(file)
+        if matched:
+            data_name = type
+            break
+
+    return data_name
+
+
+@backoff.on_exception(backoff.expo, Exception, max_value=13, max_time=34)
+def search_es(query, idx):
+    results = grq_es.search(body=query, index=idx)
+    if len(results["hits"]["hits"]) > 0:
+        return results
+    raise RuntimeError
 
 
 class OperaAccountability(Accountability):
-    def __init__(self, context):
-        Accountability.__init__(self, context)
-        self.index = INDECES[context.get("es_index", "")]
-
-    def _search(self, query):
-        """
-        searches for entries
-        :param query: query to send to elasticsearch
-        :type query: python dictionary in es format
-        :returns: list of hits
-        """
-
-        hits = []
-        try:
-            result = {}
-            if self.index in INDECES.values():
-                result = accountability_es.search(
-                    body=query, index=self.index
-                )
-            else:
-                raise Exception("Index Not Found")
-            hits = result.get("hits", {}).get("hits", [])
-        except Exception:
-            logger.warn("Failed to search this query: {}".format(query))
-        return hits
-
-    def _update_doc(self, id, body):
-        """
-        update document with given id with passed body
-        :param id: document _id
-        :type id: _id string
-        :param body: dictionary to upsert to the document
-        :type body: dictionary in the following format
-        {
-            "doc_as_upsert": True,
-            "doc": <dictionary of updated values>,
-        }
-        :returns:
-        """
-        try:
-            if self.index in INDECES.values():
-                accountability_es.update_document(
-                    id=id,
-                    index=self.index,
-                    body=body
-                )
+    def __init__(self, context, work_dir):
+        Accountability.__init__(self, context, work_dir)
+
+        self.trigger_dataset_type = context.get(oc_const.DATASET_TYPE)
+        self.trigger_dataset_id = context.get(oc_const.INPUT_DATASET_ID)
+        self.step = context.get(oc_const.STEP)
+        self.product_paths = context.get(oc_const.PRODUCT_PATHS)
+
+        self.inputs = []
+        if os.path.exists("{}/datasets.json".format(work_dir)):
+            with open("{}/datasets.json".format(work_dir), "r") as reader:
+                datasets_cfg = json.load(reader)
+
+            if isinstance(self.product_paths, list):
+                self.input_files_type = get_dataset(self.product_paths[0], datasets_cfg)
             else:
-                raise Exception("Index Not Found")
-        except Exception:
-            logger.warn("Failed to update {} with {}".format(id, body))
+                self.input_files_type = get_dataset(self.product_paths, datasets_cfg)
+        else:
+            self.input_files_type = context.get(oc_const.DATASET_TYPE)
+        if isinstance(self.product_paths, list):
+            self.inputs = list(map(lambda x: os.path.basename(x), self.product_paths))
+        else:
+            self.inputs = [os.path.basename(self.product_paths)]
+        self.output_type = PGE_STEP_DICT[self.step]
+
+    def create_job_entry(self):
+        if self.job_id is not None:
+            payload = {
+                "output_data_type": self.output_type,
+                "inputs": self.inputs,
+                "input_data_type": self.input_files_type,
+                "trigger_dataset_type": self.trigger_dataset_type,
+                "trigger_dataset_id": self.trigger_dataset_id,
+                "created_at": datetime.now().isoformat()
+            }
+            grq_es.index_document(index=oc_const.JOB_ACCOUNTABILITY_INDEX, id=self.job_id, body=payload)
+        else:
+            raise Exception("Unable to create job_accountability_catalog entry: {}".format(self.product_paths))
 
     def get_entries(self):
-        """
-        retrieves entries as a list
-        :returns: entries given a given context
-        """
-        conditions = [
-            (self.input_dataset_type + ".keyword", self.input_dataset_id)]
-        conditions = accountability_es.construct_bool_query(
-            conditions)
-        query = {"query": {"bool": {"must": conditions}}}
-
-        logger.info("Query for grabbing entries: {}".format(query))
-        accountability_docs = []
-        try:
-            hits = self._search(query)
-            logger.info("hits with .keyword count : {}".format(len(hits)))
-            accountability_docs = hits
-        except Exception:
-            logger.warn(
-                "Could not retrieve associated accountability entries\t {}: {}".format(
-                    self.input_dataset_type, self.input_dataset_id)
-            )
-        return accountability_docs
-
-    def set_products(self, products, entries=None, job_id=None):
-        if entries is None:
-            entries = self.get_entries()
+        entries = []
+        if isinstance(self.product_paths, list):
+            for input_path in self.product_paths:
+                input = os.path.basename(input_path)
+                results = grq_es.query(body={
+                                "query": {"bool": {"must": [{"term": {"_id": input}}]}}
+                            }, index="grq")
+                entries.extend(results)
+        else:
+            input = os.path.basename(self.product_paths)
+            results = grq_es.query(body={
+                            "query": {"bool": {"must": [{"term": {"_id": input}}]}}
+                        }, index="grq")
+            entries.extend(results)
+        return entries
+
+    def flatten_and_merge_accountability(self):
+        entries = self.get_entries()
+        acc = {}
         for entry in entries:
-            _id = entry.get("_id")
-            source = entry.get("_source")
-            logger.info("source: {}".format(source))
-            logger.info("step: {}".format(self.step))
-            records = []
-
-            products = list(map(
-                lambda prod: prod if "/data/work/" not in prod else os.path.basename(prod), products))
-            logger.info("products: {}".format(products))
-            source_copy = dict(source)
-
-            output_type = PGE_STEP_DICT[self.step] + "_id"
-            output_job_type = PGE_STEP_DICT[self.step] + "_job_id"
-
-            first_product = products.pop(0)
-
-            source[output_type] = first_product
-            source[output_job_type] = job_id
-            try:
-                body = {
-                    "doc_as_upsert": True,
-                    "doc": source
-                }
-                self._update_doc(_id, body)
-            except Exception:
-                logger.error(
-                    "Failed to update entry with first product {}".format(first_product)
-                )
-                return
-
-            if len(products) > 0:
-                records = []
-                for product in products:
-                    new_entry = dict(source_copy)
-                    new_entry[output_type] = product
-                    records.append(new_entry)
-                try:
-                    self.post(records)
-                except Exception:
-                    logger.error(
-                        "Failed to create new entries with newly created products {}".format(
-                            products
-                        )
-                    )
-            records.append(source)
-        return records
-
-    def set_status(self, status):
-        """
-        Function to implement a custom status setting for a specific index storing accountability statuses for associated products
-        :param status:
-        :return:
-        """
-        # if self.step not in PGE_STEP_DICT:
-        #     return
-        accountability_docs = self.get_entries()
-        if len(accountability_docs) >= 1:
-            try:
-                output_type_status = ""
-                if self.step == "L0B":
-                    output_type_status = self.step + "_status"
+            if "accountability" not in entry["_source"]["metadata"]:
+                acc_obj = {}
+            else:
+                acc_obj = entry["_source"]["metadata"]["accountability"]
+            logger.info("entry accountability object: {}".format(acc_obj))
+            for pge in acc_obj:
+                if pge in acc:
+                    if "id" in acc_obj[pge]:
+                        acc[pge]["outputs"].append(acc_obj[pge]["id"])
+                    else:
+                        acc[pge]["outputs"].extend(acc_obj[pge]["outputs"])
+
+                    if "trigger_dataset_id" in acc_obj[pge]:
+                        acc[pge]["trigger_dataset_ids"].append(acc_obj[pge]["trigger_dataset_id"])
+                    else:
+                        acc[pge]["trigger_dataset_ids"].extend(acc_obj[pge]["trigger_dataset_ids"])
+
+                    if "job_id" in acc_obj[pge]:
+                        acc[pge]["job_ids"].append(acc_obj[pge]["job_id"])
+                    else:
+                        acc[pge]["job_ids"].extend(acc_obj[pge]["job_ids"])
+
+                    acc[pge]["inputs"].extend(acc_obj[pge]["inputs"])
+
+                    acc[pge] = {
+                        "outputs": list(set(acc[pge]["outputs"])),
+                        "inputs": list(set(acc[pge]["inputs"])),
+                        "input_data_type": acc_obj[pge]["input_data_type"],
+                        "job_ids": list(set(acc[pge]["job_ids"])),
+                        "trigger_dataset_type": acc_obj[pge]["trigger_dataset_type"],
+                        "trigger_dataset_ids": list(set(acc[pge]["trigger_dataset_ids"]))
+                    }
                 else:
-                    output_type = PGE_STEP_DICT[self.step]
-                    output_type_status = output_type + "_status"
-                updated_values = {
-                    output_type_status: status,
-                    "last_modified": convert_datetime(datetime.utcnow())
+                    acc[pge] = {
+                        "inputs": acc_obj[pge]["inputs"],
+                        "input_data_type": acc_obj[pge]["input_data_type"],
+                        "trigger_dataset_type": acc_obj[pge]["trigger_dataset_type"],
+                    }
+                    if "id" in acc_obj[pge]:
+                        acc[pge]["outputs"] = [acc_obj[pge]["id"]]
+                    else:
+                        acc[pge]["outputs"] = acc_obj[pge]["outputs"]
+
+                    if "job_id" in acc_obj[pge]:
+                        acc[pge]["job_ids"] = [acc_obj[pge]["job_id"]]
+                    else:
+                        acc[pge]["job_ids"] = acc_obj[pge]["job_ids"]
+
+                    if "trigger_dataset_id" in acc_obj[pge]:
+                        acc[pge]["trigger_dataset_ids"] = [acc_obj[pge]["trigger_dataset_id"]]
+                    else:
+                        acc[pge]["trigger_dataset_ids"] = acc_obj[pge]["trigger_dataset_ids"]
+
+        logger.info("accountability obj: {}".format(acc))
+        return acc
+
+    def update_product_met_json(self, job_result):
+        work_dir = job_result.get("work_dir")
+        datasets_path = "{}/output/datasets/".format(work_dir)
+        datasets = os.listdir(datasets_path)
+        accountability_obj = self.flatten_and_merge_accountability()
+
+        for dataset in datasets:
+            output_met_json = "{}/{}/{}.met.json".format(datasets_path, dataset, dataset)
+            met_json = None
+            with open(output_met_json, "r") as f:
+                met_json = json.load(f)
+                if PGE_STEP_DICT[self.step] == "INSAR" and "ProductType" in met_json:
+                    self.output_type = met_json["ProductType"]
+                accountability_obj_copy = accountability_obj.copy()
+                accountability_obj_copy[self.output_type] = {
+                    "id": dataset,
+                    "job_id": self.job_id,
+                    "inputs": self.inputs,
+                    "input_data_type": self.input_files_type,
+                    "trigger_dataset_type": self.trigger_dataset_type,
+                    "trigger_dataset_id": self.trigger_dataset_id
                 }
-                updated_doc = {
-                    "doc_as_upsert": True,
-                    "doc": updated_values,
-                }
-                for doc in accountability_docs:
-                    self._update_doc(
-                        doc.get("_id"),
-                        updated_doc
-                    )
-            except Exception:
-                logger.warn(
-                    "Failed to update accountability docs: {}".format(
-                        traceback.format_exc()
-                    )
-                )
-        else:
-            return
+                met_json["accountability"] = accountability_obj_copy
+            with open(output_met_json, "w") as f:
+                logger.info("to write: {}".format(met_json))
+                if met_json is not None:
+                    json.dump(met_json, f)
+
+    def set_products(self, job_results):
+        self.update_product_met_json(job_result=job_results)
+        return {}
diff --git a/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml b/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml
index fc260a94b..71cdee0fb 100644
--- a/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml
+++ b/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml
@@ -32,7 +32,7 @@ runconfig:
 preconditions: [get_input_filepaths_from_state_config]
 
 # This lists all the postprocessor steps that this PGE will run after running the PGE.
-postprocess: []
+postprocess: [update_product_accountability]
 
 # For any of the precondition evaluation steps listed in the preconditions area,
 # specify function arguments here
diff --git a/opera_chimera/postprocess_functions.py b/opera_chimera/postprocess_functions.py
index 574fb66a6..83eedd617 100644
--- a/opera_chimera/postprocess_functions.py
+++ b/opera_chimera/postprocess_functions.py
@@ -4,49 +4,25 @@
 
 """
 
+from chimera.commons.constants import ChimeraConstants as chimera_consts
 from chimera.postprocess_functions import PostProcessFunctions
-import ast
-import os
-import traceback
-from datetime import datetime
-
-from util.common_util import convert_datetime
 
+from commons.es_connection import get_grq_es, get_mozart_es
+from commons.logger import logger
 from opera_chimera.accountability import OperaAccountability
 
-from commons.logger import logger
-from commons.es_connection import get_grq_es, get_mozart_es
+grq_es = get_grq_es(logger)
+mozart_es = get_mozart_es(logger)
 
-from opera_chimera.constants.opera_chimera_const import (
-    OperaChimeraConstants as oc_const,
-)
 
 class OperaPostProcessFunctions(PostProcessFunctions):
     def __init__(self, context, pge_config, settings, job_result):
-        ancillary_es = get_grq_es(logger)
-        mozart_es = get_mozart_es(logger)
         PostProcessFunctions.__init__(
-            self, context, pge_config, settings, job_result, mozart_es, ancillary_es
+            self, context, pge_config, settings, job_result, mozart_es=mozart_es, grq_es=grq_es
         )
         logger.info("job_result: {}".format(job_result))
-        self.accountability = OperaAccountability(self._context)
+        self.accountability = OperaAccountability(self._context, job_result.get(chimera_consts.WORK_DIR))
 
-    def _associate_products(self, output_types, products):
-        results = {}
-        if self.accountability.step == "L0B":
-            # going through each output_type and checking if it was created
-            for output_type in output_types:
-                # finding the search tearm ex: L0B_L_RRSD or L0B_L_HST_DRT
-                search_term = "_".join(output_type.split("_")[2:])
-                # going through each product and seing if the search term is there (HST_DRT in product_id)
-                for i in range(0, len(products)):
-                    product_id = os.path.basename(products[i])
-                    if search_term in product_id:
-                        results[output_type] = product_id
-                        products.remove(products[i])
-                        break
-            return results
-        else:
-            return {
-                output_types[0]: products[0]
-            }
\ No newline at end of file
+    def update_product_accountability(self):
+        self.accountability.set_products(self._job_result)
+        return {}
diff --git a/opera_chimera/wf_xml/L3_HLS.sf.xml b/opera_chimera/wf_xml/L3_HLS.sf.xml
index 41e377240..cd507b745 100644
--- a/opera_chimera/wf_xml/L3_HLS.sf.xml
+++ b/opera_chimera/wf_xml/L3_HLS.sf.xml
@@ -52,13 +52,13 @@
         </sf:operator>
       </sf:process>
 
-      <!--sf:process id="post_L3_HLS">
+      <sf:process id="post_L3_HLS">
         <sf:inputs>
           <sf_context/>
-          <job_id_PGE_L0B_Prime from="@#previous"/>
+          <job_id_L3_HLS from="@#previous"/>
+          <chimera_config_file>/home/ops/verdi/ops/opera-pcm/opera_chimera/configs/chimera_config.yaml</chimera_config_file>
           <pge_config_file>/home/ops/verdi/ops/opera-pcm/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml</pge_config_file>
           <settings_file>/home/ops/verdi/ops/opera-pcm/conf/settings.yaml</settings_file>
-          <chimera_config_file>/home/ops/verdi/ops/opera-pcm/opera_chimera/configs/chimera_config.yaml</chimera_config_file>
         </sf:inputs>
         <sf:outputs/>
         <sf:operator>
@@ -67,7 +67,7 @@
             <sf:binding>python:/home/ops/verdi/ops/chimera/chimera/post_processor.py?post_processor.post_process</sf:binding>
           </sf:op>
         </sf:operator>
-      </sf:process-->
+      </sf:process>
     </sf:processes>
   </sf:flow>
 </sf:sciflo>

From 7ece2ee8fbef4185cbf140ad102e0634c86f2c47 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 3 Mar 2022 12:25:25 -0800
Subject: [PATCH 02/48] feat(accountability): fixed precondition_functions.py.
 removed unnecessary accountability field.

Refs #16
---
 opera_chimera/precondition_functions.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/opera_chimera/precondition_functions.py b/opera_chimera/precondition_functions.py
index 35a4ac8fa..55b8da3a9 100644
--- a/opera_chimera/precondition_functions.py
+++ b/opera_chimera/precondition_functions.py
@@ -6,34 +6,23 @@
 import copy
 import inspect
 import json
-import re
 import os
+import re
 import traceback
+from datetime import datetime
 from typing import Dict, List
 
 import psutil
-
-#from cop import cop_catalog
-
-from datetime import datetime, timedelta
-
-from opera_chimera.accountability import OperaAccountability
-
-from commons.logger import logger
-from commons.es_connection import get_grq_es
+from chimera.precondition_functions import PreConditionFunctions
 
 from commons.constants import product_metadata
-
-from chimera.precondition_functions import PreConditionFunctions
+from commons.es_connection import get_grq_es
+from commons.logger import logger
 from opera_chimera.constants.opera_chimera_const import (
     OperaChimeraConstants as oc_const,
 )
-
-#from rost import catalog as rost_catalog
-
-from util.common_util import convert_datetime, to_datetime
+from util.common_util import convert_datetime
 from util.type_util import set_type
-#from util.stuf_util import get_stuf_info_from_xml
 
 try:
     from tools.stage_dem import main as stage_dem
@@ -55,7 +44,6 @@ class OperaPreConditionFunctions(PreConditionFunctions):
     def __init__(self, context, pge_config, settings, job_params):
         PreConditionFunctions.__init__(
             self, context, pge_config, settings, job_params)
-        self.accountability = OperaAccountability(self._context)
 
     def set_product_time(self):
         """

From e5a315db13bc7f0c32571422427c34df447d8c33 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Fri, 4 Mar 2022 12:50:03 -0800
Subject: [PATCH 03/48] feat(accountability): testing fix for wrong hysds
 version used by PCM container

Refs #16
---
 docker/Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4f0f28ab6..a4d17abe2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,7 @@
-FROM hysds/pge-base:develop
+# TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
+#FROM hysds/pge-base:develop
+ARG PGE_BASE_VERSION=v4.0.1-beta.8
+FROM hysds/pge-base:$PGE_BASE_VERSION
 
 ARG GIT_OAUTH_TOKEN
 ARG PCM_COMMONS_BRANCH

From 8a302e1e74aff9374f2a9f07a0d0e246d1add3af Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Fri, 4 Mar 2022 15:34:40 -0800
Subject: [PATCH 04/48] feat(accountability): testing fix for wrong hysds
 version used by PCM container

Refs #16
---
 docker/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a4d17abe2..0ef2e5c7f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,6 +1,7 @@
 # TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
 #FROM hysds/pge-base:develop
-ARG PGE_BASE_VERSION=v4.0.1-beta.8
+#ARG PGE_BASE_VERSION=v4.0.1-beta.8
+ARG PGE_BASE_VERSION=develop
 FROM hysds/pge-base:$PGE_BASE_VERSION
 
 ARG GIT_OAUTH_TOKEN

From 7d0d31916c4ff5f4610f1e5b8f873da88307ece2 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Fri, 4 Mar 2022 16:07:42 -0800
Subject: [PATCH 05/48] feat(accountability): testing fix for wrong hysds
 version used by PCM container

Refs #16
---
 docker/Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 0ef2e5c7f..d6c965515 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,8 +1,8 @@
 # TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
 #FROM hysds/pge-base:develop
-#ARG PGE_BASE_VERSION=v4.0.1-beta.8
-ARG PGE_BASE_VERSION=develop
-FROM hysds/pge-base:$PGE_BASE_VERSION
+ARG PGE_BASE_VERSION=v4.0.1-beta.8
+#ARG PGE_BASE_VERSION=develop
+FROM hysds/pge-base:${PGE_BASE_VERSION}
 
 ARG GIT_OAUTH_TOKEN
 ARG PCM_COMMONS_BRANCH

From fb9ba0d2df524dd1527fac2e4d819466e9482ffd Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 09:32:54 -0800
Subject: [PATCH 06/48] feat(accountability): testing hardcoded value for
 module_path sciflo job positional parameter.

This value is added to PYTHONPATH in run_sciflo.sh in chimera.

Refs #16
---
 docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG b/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
index 151a02073..41eb10b82 100644
--- a/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
+++ b/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
@@ -16,6 +16,12 @@
       "name":"dataset_type",
       "type":"text",
       "from":"dataset_jpath:_source.dataset"
+    },
+    {
+      "name": "module_path",
+      "from": "value",
+      "type": "text",
+      "value": "/home/ops/verdi/ops/opera-pcm"
     }
   ]
 }
\ No newline at end of file

From a5bb3bf972f71450a2ac76807f1521e5cf22065f Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 10:02:09 -0800
Subject: [PATCH 07/48] feat(accountability): committing Pycharm files

---
 .idea/misc.xml      |  4 ++++
 .idea/modules.xml   |  8 ++++++++
 .idea/opera-pcm.iml | 12 ++++++++++++
 .idea/vcs.xml       |  6 ++++++
 4 files changed, 30 insertions(+)
 create mode 100644 .idea/misc.xml
 create mode 100644 .idea/modules.xml
 create mode 100644 .idea/opera-pcm.iml
 create mode 100644 .idea/vcs.xml

diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 000000000..090d07eed
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (opera-sds-pcm)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 000000000..0244d5e47
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/opera-pcm.iml" filepath="$PROJECT_DIR$/.idea/opera-pcm.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/opera-pcm.iml b/.idea/opera-pcm.iml
new file mode 100644
index 000000000..95e658f73
--- /dev/null
+++ b/.idea/opera-pcm.iml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/chimera/chimera" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.8 (opera-sds-pcm)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..94a25f7f4
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

From d0ca3480bb654307c98f2d6f75505d42b323321b Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 10:03:52 -0800
Subject: [PATCH 08/48] feat(accountability): revert

---
 docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG b/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
index 41eb10b82..151a02073 100644
--- a/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
+++ b/docker/hysds-io.json.L3_DSWx_HLS_STATE_CONFIG
@@ -16,12 +16,6 @@
       "name":"dataset_type",
       "type":"text",
       "from":"dataset_jpath:_source.dataset"
-    },
-    {
-      "name": "module_path",
-      "from": "value",
-      "type": "text",
-      "value": "/home/ops/verdi/ops/opera-pcm"
     }
   ]
 }
\ No newline at end of file

From 3e355a162cd0c0bf3ae964b7b7ec84e8c5e629e0 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 10:04:40 -0800
Subject: [PATCH 09/48] feat(accountability): pycharm

---
 .idea/opera-pcm.iml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.idea/opera-pcm.iml b/.idea/opera-pcm.iml
index 95e658f73..452d5ea8d 100644
--- a/.idea/opera-pcm.iml
+++ b/.idea/opera-pcm.iml
@@ -3,7 +3,7 @@
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/chimera/chimera" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/commons" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.8 (opera-sds-pcm)" jdkType="Python SDK" />

From bb59ee12fb73e22be3eb7c1ed197c7cada6651a8 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 11:04:37 -0800
Subject: [PATCH 10/48] feat(accountability): testing `beta.2` reference
 instead of develop for `hysds/pge-base` docker image

should behave similarly to outdated `develop` reference.

Refs #16
---
 docker/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index d6c965515..b7d11d803 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,6 +1,7 @@
 # TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
 #FROM hysds/pge-base:develop
-ARG PGE_BASE_VERSION=v4.0.1-beta.8
+ARG PGE_BASE_VERSION=v4.0.1-beta.2
+#ARG PGE_BASE_VERSION=v4.0.1-beta.8
 #ARG PGE_BASE_VERSION=develop
 FROM hysds/pge-base:${PGE_BASE_VERSION}
 

From a57a7ead7c75f892f71a2bebdcc4ffe36b608839 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 13:09:28 -0800
Subject: [PATCH 11/48] feat(accountability): testing `beta.8` reference
 instead of develop for `hysds/pge-base` docker image

Refs #16
---
 docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index b7d11d803..d4a42b1c3 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,7 +1,7 @@
 # TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
 #FROM hysds/pge-base:develop
-ARG PGE_BASE_VERSION=v4.0.1-beta.2
-#ARG PGE_BASE_VERSION=v4.0.1-beta.8
+#ARG PGE_BASE_VERSION=v4.0.1-beta.2
+ARG PGE_BASE_VERSION=v4.0.1-beta.8
 #ARG PGE_BASE_VERSION=develop
 FROM hysds/pge-base:${PGE_BASE_VERSION}
 

From d505a16db51875b39b0c7e847cc78f57402f40ab Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 13:35:14 -0800
Subject: [PATCH 12/48] feat(accountability): adding accountability params

Refs #16
---
 docker/hysds-io.json.SCIFLO_L3_DSWx_HLS | 14 +++++++++++++-
 docker/job-spec.json.SCIFLO_L3_DSWx_HLS | 10 +++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
index 3fede9ac2..fbb6bf3ec 100644
--- a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
@@ -35,6 +35,18 @@
      "name": "product_metadata",
      "from": "dataset_jpath:_source",
      "lambda" : "lambda ds: { 'metadata': ds['metadata'] }"
-    }
+    },
+    {
+      "name": "accountability_module_path",
+      "from": "value",
+      "type": "text",
+      "value": "opera_chimera.accountability"
+    },
+    {
+      "name": "accountability_class",
+      "from": "value",
+      "type": "text",
+      "value": "OperaAccountability"
+    },
   ]
 }
diff --git a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
index 1b69df895..327b5a1d0 100644
--- a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
@@ -44,6 +44,14 @@
     {
       "name": "product_metadata",
       "destination": "context"
-    }
+    },
+    {
+      "name": "accountability_module_path",
+      "destination": "context"
+    },
+    {
+      "name": "accountability_class",
+      "destination": "context"
+    },
   ]
 }

From 62f0415495fa67f64d03c7eb013bb60a01a87738 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 13:35:58 -0800
Subject: [PATCH 13/48] feat(accountability): adding accountability params

Refs #16
---
 docker/hysds-io.json.SCIFLO_L3_DSWx_HLS | 2 +-
 docker/job-spec.json.SCIFLO_L3_DSWx_HLS | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
index fbb6bf3ec..851a04c35 100644
--- a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
@@ -47,6 +47,6 @@
       "from": "value",
       "type": "text",
       "value": "OperaAccountability"
-    },
+    }
   ]
 }
diff --git a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
index 327b5a1d0..4cbfa3b94 100644
--- a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
@@ -52,6 +52,6 @@
     {
       "name": "accountability_class",
       "destination": "context"
-    },
+    }
   ]
 }

From e2a03ae4e5f9695693b1f0bceec3cb8ac552dae8 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 13:36:37 -0800
Subject: [PATCH 14/48] feat(accountability): testing without conda stuff to
 speed up jenkins builds

conda stuff might be unnecessary.

Refs #16
---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index d4a42b1c3..d1401a80c 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -15,7 +15,7 @@ COPY . /home/ops/verdi/ops/opera-pcm
 RUN set -ex \
  && source /home/ops/.bash_profile \
  && pip install -U pytest scripttest mock mockito flake8 pytest-cov flake8-junit-report flake8-string-format xmltodict yamale==3.0.6 ruamel.yaml elasticmock geopandas smart_open \
- && sudo /opt/conda/bin/conda install -y gdal cffi \
+# && sudo /opt/conda/bin/conda install -y gdal cffi \
  && sudo chown -R ops:ops /home/ops/verdi/ops/opera-pcm \
  && cd /home/ops/verdi/ops \
  && git clone https://${GIT_OAUTH_TOKEN}@github.jpl.nasa.gov/IEMS-SDS/pcm_commons.git \

From aa1c6c6abacd6afbb312f9761cced1891c34e838 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 14:57:27 -0800
Subject: [PATCH 15/48] feat(accountability): testing accountability with
 hardcoded output type.

Refs #16
---
 opera_chimera/accountability.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index 578d75658..1245fdd04 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -64,7 +64,7 @@ def __init__(self, context, work_dir):
 
         self.trigger_dataset_type = context.get(oc_const.DATASET_TYPE)
         self.trigger_dataset_id = context.get(oc_const.INPUT_DATASET_ID)
-        self.step = context.get(oc_const.STEP)
+        # self.step = context.get(oc_const.STEP)  # TODO chrisjrd: resolve
         self.product_paths = context.get(oc_const.PRODUCT_PATHS)
 
         self.inputs = []
@@ -82,7 +82,13 @@ def __init__(self, context, work_dir):
             self.inputs = list(map(lambda x: os.path.basename(x), self.product_paths))
         else:
             self.inputs = [os.path.basename(self.product_paths)]
-        self.output_type = PGE_STEP_DICT[self.step]
+        # TODO chrisjrd: resolve
+        #  Use:
+        #  * wf_name/purpose job param from job-spec
+        #  * PGE output type from PGE config YAML
+        #  * .sf.xml name (workflow name / wf_name?)
+        # self.output_type = PGE_STEP_DICT[self.step]
+        self.output_type = "L3_DSWx"  # got this from PGE config YAML
 
     def create_job_entry(self):
         if self.job_id is not None:
@@ -186,8 +192,6 @@ def update_product_met_json(self, job_result):
             met_json = None
             with open(output_met_json, "r") as f:
                 met_json = json.load(f)
-                if PGE_STEP_DICT[self.step] == "INSAR" and "ProductType" in met_json:
-                    self.output_type = met_json["ProductType"]
                 accountability_obj_copy = accountability_obj.copy()
                 accountability_obj_copy[self.output_type] = {
                     "id": dataset,

From c91fb55a61f82a965be9dccd2209573ee815f48d Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 15:59:12 -0800
Subject: [PATCH 16/48] feat(accountability): testing accountability with
 hardcoded output type.

Refs #16
---
 opera_chimera/accountability.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index 1245fdd04..4ebfcefc7 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -1,3 +1,5 @@
+from typing import Dict, List
+
 import backoff
 import re
 import json
@@ -65,19 +67,24 @@ def __init__(self, context, work_dir):
         self.trigger_dataset_type = context.get(oc_const.DATASET_TYPE)
         self.trigger_dataset_id = context.get(oc_const.INPUT_DATASET_ID)
         # self.step = context.get(oc_const.STEP)  # TODO chrisjrd: resolve
-        self.product_paths = context.get(oc_const.PRODUCT_PATHS)
+        # self.product_paths = context.get(oc_const.PRODUCT_PATHS)
 
-        self.inputs = []
-        if os.path.exists("{}/datasets.json".format(work_dir)):
-            with open("{}/datasets.json".format(work_dir), "r") as reader:
-                datasets_cfg = json.load(reader)
+        metadata: Dict[str, str] = context["product_metadata"]["metadata"]
+        self.product_paths: List[str] = [product_path for band_or_qa, product_path in metadata.items() if band_or_qa != '@timestamp']
 
-            if isinstance(self.product_paths, list):
-                self.input_files_type = get_dataset(self.product_paths[0], datasets_cfg)
-            else:
-                self.input_files_type = get_dataset(self.product_paths, datasets_cfg)
-        else:
-            self.input_files_type = context.get(oc_const.DATASET_TYPE)
+        self.inputs = []
+        # TODO chrisjrd: resolve getting dataset type associated with the input files
+        # if os.path.exists("{}/datasets.json".format(work_dir)):
+        #     with open("{}/datasets.json".format(work_dir), "r") as reader:
+        #         datasets_cfg = json.load(reader)
+        #
+        #     if isinstance(self.product_paths, list):
+        #         self.input_files_type = get_dataset(self.product_paths[0], datasets_cfg)
+        #     else:
+        #         self.input_files_type = get_dataset(self.product_paths, datasets_cfg)
+        # else:
+        #     self.input_files_type = context.get(oc_const.DATASET_TYPE)
+        self.input_files_type = "TBD"  # TODO chrisjrd: resolve
         if isinstance(self.product_paths, list):
             self.inputs = list(map(lambda x: os.path.basename(x), self.product_paths))
         else:

From 78bf26e8424dbbb3aa9e621fc778d25b2618707c Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Mon, 7 Mar 2022 16:30:50 -0800
Subject: [PATCH 17/48] feat(accountability): adding missing const for
 accountability Elasticsearch index

Refs #16
---
 opera_chimera/constants/opera_chimera_const.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/opera_chimera/constants/opera_chimera_const.py b/opera_chimera/constants/opera_chimera_const.py
index 2bca5749c..244e44264 100644
--- a/opera_chimera/constants/opera_chimera_const.py
+++ b/opera_chimera/constants/opera_chimera_const.py
@@ -1,4 +1,5 @@
 from chimera.commons.constants import ChimeraConstants
+from job_accountability.catalog import ES_INDEX as job_accountability_index
 #from pass_accountability.catalog import ES_INDEX as pass_accountability_index
 #from observation_accountability.catalog import ES_INDEX as observation_accountability_index
 #from Track_Frame_Accountability.catalog import ES_INDEX as track_frame_accountability_index
@@ -144,6 +145,7 @@ def __init__(self):
     #L0B = "L0B"
 
     # Other Constants
+    JOB_ACCOUNTABILITY_INDEX = job_accountability_index
     #PASS_ACCOUNTABILITY_INDEX = pass_accountability_index
     #OBSERVATION_ACCOUNTABILITY_INDEX = observation_accountability_index
     #TRACK_FRAME_ACCOUNTABILITY_INDEX = track_frame_accountability_index

From 816386ce48ee714bda946dd659b348e2030c5fe7 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Tue, 8 Mar 2022 09:59:22 -0800
Subject: [PATCH 18/48] feat(accountability): testing accountability logic with
 hardcoded output path

Refs #16
---
 opera_chimera/accountability.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index 4ebfcefc7..279ec2386 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -190,7 +190,7 @@ def flatten_and_merge_accountability(self):
 
     def update_product_met_json(self, job_result):
         work_dir = job_result.get("work_dir")
-        datasets_path = "{}/output/datasets/".format(work_dir)
+        datasets_path = f"{work_dir}/output_dir_tbf/datasets/"  # TODO chrisjrd: resolve
         datasets = os.listdir(datasets_path)
         accountability_obj = self.flatten_and_merge_accountability()
 

From 62990596fb29976f915f852955de198100c903af Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Tue, 8 Mar 2022 15:13:46 -0800
Subject: [PATCH 19/48] feat(accountability): testing build parameter for
 Jenkins CI job that builds OPERA PCM as a docker container.

Refs #16
---
 conf/sds/files/config.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/conf/sds/files/config.xml b/conf/sds/files/config.xml
index 9a8b1e5bd..5e22a60e7 100644
--- a/conf/sds/files/config.xml
+++ b/conf/sds/files/config.xml
@@ -24,6 +24,12 @@
           <defaultValue>{{ PCM_COMMONS_BRANCH }}</defaultValue>
           <trim>true</trim>
         </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>PGE_BASE_VERSION</name>
+          <description>The hysds/pge-base image version to use</description>
+          <defaultValue>{{ VERDI_TAG }}</defaultValue>
+          <trim>true</trim>
+        </hudson.model.StringParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
   </properties>

From e8c69a6652750b9d4cd589fc4597ed308bb78369 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Tue, 8 Mar 2022 15:37:13 -0800
Subject: [PATCH 20/48] feat(accountability): testing build parameter for
 Jenkins CI job that builds OPERA PCM as a docker container.

Refs #16
---
 conf/sds/files/config-branch.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/conf/sds/files/config-branch.xml b/conf/sds/files/config-branch.xml
index 9de70c0e0..27a527036 100644
--- a/conf/sds/files/config-branch.xml
+++ b/conf/sds/files/config-branch.xml
@@ -24,6 +24,12 @@
           <defaultValue>{{ PCM_COMMONS_BRANCH }}</defaultValue>
           <trim>true</trim>
         </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>PGE_BASE_VERSION</name>
+          <description>The hysds/pge-base image version to use</description>
+          <defaultValue>{{ VERDI_TAG }}</defaultValue>
+          <trim>true</trim>
+        </hudson.model.StringParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
   </properties>

From 0a584ce6f91cd589af3ec325754c71908897959a Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Tue, 8 Mar 2022 16:15:03 -0800
Subject: [PATCH 21/48] feat(accountability): passing hysds/pge-base version
 jenkins build parameter to the execute shell command

Refs #16
---
 conf/sds/files/config-branch.xml | 12 ++++++++++--
 conf/sds/files/config.xml        | 12 ++++++++++--
 docker/Dockerfile                |  7 +------
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/conf/sds/files/config-branch.xml b/conf/sds/files/config-branch.xml
index 27a527036..f96f13eef 100644
--- a/conf/sds/files/config-branch.xml
+++ b/conf/sds/files/config-branch.xml
@@ -89,10 +89,18 @@ set -ex
 if [ ! -z "$CONTAINER_REGISTRY" -a ! -z "$CONTAINER_REGISTRY_BUCKET" ]
 then
   {{ OPS_HOME }}/verdi/ops/container-builder/build-container.bash \
-  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} ${CONTAINER_REGISTRY} --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} --build-arg BRANCH=${TAG} --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} 
+  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} ${CONTAINER_REGISTRY} \
+        --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} \
+        --build-arg BRANCH=${TAG} \
+        --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} \
+        --build-arg PGE_BASE_VERSION=${PGE_BASE_VERSION}
 else
   {{ OPS_HOME }}/verdi/ops/container-builder/build-container.bash \
-  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} "" --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} --build-arg BRANCH=${TAG} --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} 
+  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} "" \
+        --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} \
+        --build-arg BRANCH=${TAG} \
+        --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} \
+        --build-arg PGE_BASE_VERSION=${PGE_BASE_VERSION}
 fi
 
 if [ -f "./docker/extract_artifacts.sh" ]; then
diff --git a/conf/sds/files/config.xml b/conf/sds/files/config.xml
index 5e22a60e7..d4d75c70b 100644
--- a/conf/sds/files/config.xml
+++ b/conf/sds/files/config.xml
@@ -90,10 +90,18 @@ set -ex
 if [ ! -z "$CONTAINER_REGISTRY" -a ! -z "$CONTAINER_REGISTRY_BUCKET" ]
 then
   {{ OPS_HOME }}/verdi/ops/container-builder/build-container.bash \
-  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} ${CONTAINER_REGISTRY} --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} --build-arg BRANCH=${TAG} --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} 
+  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} ${CONTAINER_REGISTRY} \
+        --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} \
+        --build-arg BRANCH=${TAG} \
+        --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} \
+        --build-arg PGE_BASE_VERSION=${PGE_BASE_VERSION}
 else
   {{ OPS_HOME }}/verdi/ops/container-builder/build-container.bash \
-  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} "" --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} --build-arg BRANCH=${TAG} --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} 
+  ${REPO} ${TAG} ${STORAGE} ${MOZART_REST_URL} ${GRQ_REST_URL} ${SKIP_PUBLISH} "" \
+        --build-arg GIT_OAUTH_TOKEN=${GIT_OAUTH_TOKEN} \
+        --build-arg BRANCH=${TAG} \
+        --build-arg PCM_COMMONS_BRANCH=${PCM_COMMONS_BRANCH} \
+        --build-arg PGE_BASE_VERSION=${PGE_BASE_VERSION}
 fi
 
 if [ -f "./docker/extract_artifacts.sh" ]; then
diff --git a/docker/Dockerfile b/docker/Dockerfile
index d1401a80c..ad97bef1e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,8 +1,4 @@
-# TODO chrisjrd: finalize. testing out setting docker image version other than "develop"
-#FROM hysds/pge-base:develop
-#ARG PGE_BASE_VERSION=v4.0.1-beta.2
-ARG PGE_BASE_VERSION=v4.0.1-beta.8
-#ARG PGE_BASE_VERSION=develop
+ARG PGE_BASE_VERSION
 FROM hysds/pge-base:${PGE_BASE_VERSION}
 
 ARG GIT_OAUTH_TOKEN
@@ -15,7 +11,6 @@ COPY . /home/ops/verdi/ops/opera-pcm
 RUN set -ex \
  && source /home/ops/.bash_profile \
  && pip install -U pytest scripttest mock mockito flake8 pytest-cov flake8-junit-report flake8-string-format xmltodict yamale==3.0.6 ruamel.yaml elasticmock geopandas smart_open \
-# && sudo /opt/conda/bin/conda install -y gdal cffi \
  && sudo chown -R ops:ops /home/ops/verdi/ops/opera-pcm \
  && cd /home/ops/verdi/ops \
  && git clone https://${GIT_OAUTH_TOKEN}@github.jpl.nasa.gov/IEMS-SDS/pcm_commons.git \

From 0d303efd5294ed01273aab19c7b65309b6d96bd5 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Tue, 8 Mar 2022 16:23:57 -0800
Subject: [PATCH 22/48] feat(accountability): Bach branch names now default to
 `develop`

Refs #16
---
 cluster_provisioning/dev/variables.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cluster_provisioning/dev/variables.tf b/cluster_provisioning/dev/variables.tf
index a70d121bc..602e79932 100644
--- a/cluster_provisioning/dev/variables.tf
+++ b/cluster_provisioning/dev/variables.tf
@@ -48,7 +48,7 @@ variable "bach_api_repo" {
 }
 
 variable "bach_api_branch" {
-  default = "main"
+  default = "develop"
 }
 
 variable "bach_ui_repo" {
@@ -56,7 +56,7 @@ variable "bach_ui_repo" {
 }
 
 variable "bach_ui_branch" {
-  default = "main"
+  default = "develop"
 }
 
 variable "venue" {

From 049ee33663fbe79527793f8cd0362c8968c32f6c Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Wed, 9 Mar 2022 11:06:56 -0800
Subject: [PATCH 23/48] feat(terraform): updated terraform minimum version
 reference. updated terraform AWS provider reference.

the terraform version reference was out of date, given the syntax used in the same versions.tf file where it is used.

updated the AWS provider to latest 2.x worked flawlessly, while updating to latest 3.x presented issues during dev testing.

Refs #16
---
 cluster_provisioning/death-valley/versions.tf |  6 ++--
 .../dev-e2e-baseline-pge/versions.tf          |  4 +--
 .../dev-e2e-event-misfire/versions.tf         |  5 ++--
 cluster_provisioning/dev-e2e-pge/versions.tf  |  4 +--
 cluster_provisioning/dev-int/versions.tf      |  4 +--
 cluster_provisioning/dev-releaser/versions.tf |  4 +--
 cluster_provisioning/dev/.terraform.lock.hcl  | 28 +++++++++----------
 cluster_provisioning/dev/versions.tf          |  4 +--
 .../modules/common/versions.tf                |  6 ++--
 9 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/cluster_provisioning/death-valley/versions.tf b/cluster_provisioning/death-valley/versions.tf
index eb9753b83..e465dbaa8 100644
--- a/cluster_provisioning/death-valley/versions.tf
+++ b/cluster_provisioning/death-valley/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
-      source = "hashicorp/aws"
-      version = "2.63"
+      source  = "hashicorp/aws"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev-e2e-baseline-pge/versions.tf b/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
index 3a6011bc7..e465dbaa8 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.63"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev-e2e-event-misfire/versions.tf b/cluster_provisioning/dev-e2e-event-misfire/versions.tf
index f6734bd6a..304e8bb72 100644
--- a/cluster_provisioning/dev-e2e-event-misfire/versions.tf
+++ b/cluster_provisioning/dev-e2e-event-misfire/versions.tf
@@ -1,9 +1,10 @@
 
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
   required_providers {
     aws = {
-      source = "hashicorp/aws"
+      source  = "hashicorp/aws"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev-e2e-pge/versions.tf b/cluster_provisioning/dev-e2e-pge/versions.tf
index 3a6011bc7..e465dbaa8 100644
--- a/cluster_provisioning/dev-e2e-pge/versions.tf
+++ b/cluster_provisioning/dev-e2e-pge/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.63"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev-int/versions.tf b/cluster_provisioning/dev-int/versions.tf
index 3a6011bc7..e465dbaa8 100644
--- a/cluster_provisioning/dev-int/versions.tf
+++ b/cluster_provisioning/dev-int/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.63"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev-releaser/versions.tf b/cluster_provisioning/dev-releaser/versions.tf
index 3a6011bc7..e465dbaa8 100644
--- a/cluster_provisioning/dev-releaser/versions.tf
+++ b/cluster_provisioning/dev-releaser/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.63"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/dev/.terraform.lock.hcl b/cluster_provisioning/dev/.terraform.lock.hcl
index c226fe074..72258ecff 100644
--- a/cluster_provisioning/dev/.terraform.lock.hcl
+++ b/cluster_provisioning/dev/.terraform.lock.hcl
@@ -2,22 +2,20 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/aws" {
-  version     = "2.63.0"
-  constraints = "2.63.0"
+  version     = "2.70.1"
+  constraints = "2.70.1"
   hashes = [
-    "h1:sBNeljIbf5YqWtnJXbNAIFKZgHWfYWJKczl1Lt6ArcQ=",
-    "zh:0526f1443ee3609a9d670e7bdded1f23622c44c12434cd76a064faf686c7244f",
-    "zh:05f2c987a8c4b18bb1133cacdc950955b11e51f3ed95667687ca2013e1ea622d",
-    "zh:2f033d6da4d4d66ab4b0f4e8fd3d55ed27ca9d146ccd3a644d50218c08b3782e",
-    "zh:39ebaa3288eb902c889eae8ac23e4fb4c75f564ae7e6dd64024f39f280ff83e4",
-    "zh:3a79f811318375126cf4a76d4dac6be8817a7d9cab6ae016e4604025f34ad5fe",
-    "zh:583d48364b6abcfafe5e0ca568bd447822fb55f39dbf56c4ae6fb4b7231be288",
-    "zh:7ae37141b5ba4d29fceb6a9725b80bd66cfd37756e772c27d07310c9a030be59",
-    "zh:93b4047654bcc9975984e6dfbbdd5912c44b8607cb1bf0da159ebbf2bc301448",
-    "zh:9cdedbf61e8adedd35203bebe4d874b7eb345105dad2360a6ef342985a9b45cb",
-    "zh:cba668d453476a8ba580d26f9e72574e164080333793a76702329ccc5f7561af",
-    "zh:de425790e68e5acab199deacc8b45df4e34b8138ae38e793b323c051580ce79e",
-    "zh:ea079694358ac9e3c111a4af4f3ba79785af13e1337e0484fb04628464c86983",
+    "h1:B+Cs8HwWnLn9ymYI/r3B5S10w1+hOD5WmfFxiLHTUIA=",
+    "zh:04137cdf128cf21dcd190bbba4d4bba43c7868c52ad646b0eaa54a8b8b8160a7",
+    "zh:30c9f956133a102b4a426d76dd3ef1a42332d9875261a06aa877409aa6b2b556",
+    "zh:3107a43647454a3d6d847fba6aa593650af0f6a353272c04450408af5f4d353a",
+    "zh:3f17285478313af822447b453fa4e37f30ef221f0b0e8f2e4655f1ac9f9de1a2",
+    "zh:5a626f7a3c4a9fea3bdfde63aedbf6eea73760f3b228f776f1132b61d00c7ff2",
+    "zh:6aafc9dd79b511b9e3d0ec49f7df1d1fd697c3c873d1d70a2be1a12475b50206",
+    "zh:6fb29b48ccc85f7e9dfde3867ce99d6d65fb76bea68c97d404fae431758a8f03",
+    "zh:c47be92e1edf2e8675c932030863536c1a79decf85b2baa4232e5936c5f7088f",
+    "zh:cd0a4b28c5e4b5092043803d17fd1d495ecb926c2688603c4cdab4c20f3a91f4",
+    "zh:fb0ff763cb5d7a696989e58e0e4b88b1faed2a62b9fb83f4f7c2400ad6fabb84",
   ]
 }
 
diff --git a/cluster_provisioning/dev/versions.tf b/cluster_provisioning/dev/versions.tf
index 3a6011bc7..e465dbaa8 100644
--- a/cluster_provisioning/dev/versions.tf
+++ b/cluster_provisioning/dev/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.63"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"
diff --git a/cluster_provisioning/modules/common/versions.tf b/cluster_provisioning/modules/common/versions.tf
index eb9753b83..e465dbaa8 100644
--- a/cluster_provisioning/modules/common/versions.tf
+++ b/cluster_provisioning/modules/common/versions.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1.0.11"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
-      source = "hashicorp/aws"
-      version = "2.63"
+      source  = "hashicorp/aws"
+      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"

From c8f50d8fbf312fa8623ddb54d81e38f8dce21b47 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 10 Mar 2022 09:36:40 -0800
Subject: [PATCH 24/48] feat(accountability): cleaned up accountability code

Refs #16
---
 opera_chimera/accountability.py | 93 +++++++++++----------------------
 1 file changed, 30 insertions(+), 63 deletions(-)

diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index 279ec2386..1a7c38829 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict
 
 import backoff
 import re
@@ -20,11 +20,6 @@
 grq_es = get_job_accountability_connection(logger)
 
 
-PGE_STEP_DICT = {
-    # "L0A": "L0A_L_RRST_PP"
-}
-
-
 def get_dataset(key, datasets_cfg):
     dataset = None
     for ds in datasets_cfg["datasets"]:
@@ -61,40 +56,18 @@ def search_es(query, idx):
 
 
 class OperaAccountability(Accountability):
-    def __init__(self, context, work_dir):
+    def __init__(self, context: Dict, work_dir: str):
         Accountability.__init__(self, context, work_dir)
 
-        self.trigger_dataset_type = context.get(oc_const.DATASET_TYPE)
-        self.trigger_dataset_id = context.get(oc_const.INPUT_DATASET_ID)
-        # self.step = context.get(oc_const.STEP)  # TODO chrisjrd: resolve
-        # self.product_paths = context.get(oc_const.PRODUCT_PATHS)
+        self.trigger_dataset_type = context[oc_const.DATASET_TYPE]
+        self.trigger_dataset_id = context[oc_const.INPUT_DATASET_ID]
 
         metadata: Dict[str, str] = context["product_metadata"]["metadata"]
-        self.product_paths: List[str] = [product_path for band_or_qa, product_path in metadata.items() if band_or_qa != '@timestamp']
-
-        self.inputs = []
-        # TODO chrisjrd: resolve getting dataset type associated with the input files
-        # if os.path.exists("{}/datasets.json".format(work_dir)):
-        #     with open("{}/datasets.json".format(work_dir), "r") as reader:
-        #         datasets_cfg = json.load(reader)
-        #
-        #     if isinstance(self.product_paths, list):
-        #         self.input_files_type = get_dataset(self.product_paths[0], datasets_cfg)
-        #     else:
-        #         self.input_files_type = get_dataset(self.product_paths, datasets_cfg)
-        # else:
-        #     self.input_files_type = context.get(oc_const.DATASET_TYPE)
-        self.input_files_type = "TBD"  # TODO chrisjrd: resolve
-        if isinstance(self.product_paths, list):
-            self.inputs = list(map(lambda x: os.path.basename(x), self.product_paths))
-        else:
-            self.inputs = [os.path.basename(self.product_paths)]
-        # TODO chrisjrd: resolve
-        #  Use:
-        #  * wf_name/purpose job param from job-spec
-        #  * PGE output type from PGE config YAML
-        #  * .sf.xml name (workflow name / wf_name?)
-        # self.output_type = PGE_STEP_DICT[self.step]
+        self.product_paths = [product_path for band_or_qa, product_path in metadata.items() if band_or_qa != '@timestamp']  # TODO chrisjrd: improve dataset structure to avoid duplicating this logic from eval_state_config.py
+
+        self.input_files_type = self.remove_suffix(self.trigger_dataset_type, "-state-config")
+        self.inputs = [os.path.basename(product_path) for product_path in self.product_paths]
+
         self.output_type = "L3_DSWx"  # got this from PGE config YAML
 
     def create_job_entry(self):
@@ -113,15 +86,8 @@ def create_job_entry(self):
 
     def get_entries(self):
         entries = []
-        if isinstance(self.product_paths, list):
-            for input_path in self.product_paths:
-                input = os.path.basename(input_path)
-                results = grq_es.query(body={
-                                "query": {"bool": {"must": [{"term": {"_id": input}}]}}
-                            }, index="grq")
-                entries.extend(results)
-        else:
-            input = os.path.basename(self.product_paths)
+        for input_path in self.product_paths:
+            input = os.path.basename(input_path)
             results = grq_es.query(body={
                             "query": {"bool": {"must": [{"term": {"_id": input}}]}}
                         }, index="grq")
@@ -189,30 +155,31 @@ def flatten_and_merge_accountability(self):
         return acc
 
     def update_product_met_json(self, job_result):
+        """Creates a .met.json with updated accountability metadata."""
         work_dir = job_result.get("work_dir")
         datasets_path = f"{work_dir}/output_dir_tbf/datasets/"  # TODO chrisjrd: resolve
         datasets = os.listdir(datasets_path)
-        accountability_obj = self.flatten_and_merge_accountability()
+        old_accountability = self.flatten_and_merge_accountability()
 
         for dataset in datasets:
-            output_met_json = "{}/{}/{}.met.json".format(datasets_path, dataset, dataset)
-            met_json = None
-            with open(output_met_json, "r") as f:
+            new_accountability = old_accountability.copy()
+            new_accountability[self.output_type] = {
+                "id": dataset,
+                "job_id": self.job_id,
+                "inputs": self.inputs,
+                "input_data_type": self.input_files_type,
+                "trigger_dataset_type": self.trigger_dataset_type,
+                "trigger_dataset_id": self.trigger_dataset_id
+            }
+
+            output_met_json_filepath = f"{datasets_path}/{dataset}/{dataset}.met.json"
+            with open(output_met_json_filepath, "r") as f:
                 met_json = json.load(f)
-                accountability_obj_copy = accountability_obj.copy()
-                accountability_obj_copy[self.output_type] = {
-                    "id": dataset,
-                    "job_id": self.job_id,
-                    "inputs": self.inputs,
-                    "input_data_type": self.input_files_type,
-                    "trigger_dataset_type": self.trigger_dataset_type,
-                    "trigger_dataset_id": self.trigger_dataset_id
-                }
-                met_json["accountability"] = accountability_obj_copy
-            with open(output_met_json, "w") as f:
-                logger.info("to write: {}".format(met_json))
-                if met_json is not None:
-                    json.dump(met_json, f)
+                met_json["accountability"] = new_accountability
+
+            with open(output_met_json_filepath, "w") as f:
+                logger.info(f"to write: {met_json}")
+                json.dump(met_json, f)
 
     def set_products(self, job_results):
         self.update_product_met_json(job_result=job_results)

From 6b4fbddba6c3f1eb3d63ebd5249c9d7245acb66f Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 10 Mar 2022 12:11:20 -0800
Subject: [PATCH 25/48] feat(accountability): cleaned up accountability code

Refs #16
---
 opera_chimera/accountability.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index 1a7c38829..ae4c4ff7d 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -55,6 +55,13 @@ def search_es(query, idx):
     raise RuntimeError
 
 
+def remove_suffix(input_string: str, suffix: str) -> str:
+    """Polyfill for str.removesuffix function introduced in Python 3.9."""
+    if suffix and input_string.endswith(suffix):
+        return input_string[:-len(suffix)]
+    return input_string
+
+
 class OperaAccountability(Accountability):
     def __init__(self, context: Dict, work_dir: str):
         Accountability.__init__(self, context, work_dir)
@@ -65,7 +72,7 @@ def __init__(self, context: Dict, work_dir: str):
         metadata: Dict[str, str] = context["product_metadata"]["metadata"]
         self.product_paths = [product_path for band_or_qa, product_path in metadata.items() if band_or_qa != '@timestamp']  # TODO chrisjrd: improve dataset structure to avoid duplicating this logic from eval_state_config.py
 
-        self.input_files_type = self.remove_suffix(self.trigger_dataset_type, "-state-config")
+        self.input_files_type = remove_suffix(self.trigger_dataset_type, "-state-config")
         self.inputs = [os.path.basename(product_path) for product_path in self.product_paths]
 
         self.output_type = "L3_DSWx"  # got this from PGE config YAML

From 34adb00ee3c42079e44bbea076b3a29f16bb1a77 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 10 Mar 2022 12:46:22 -0800
Subject: [PATCH 26/48] feat(accountability): syncing some Terraform script
 improvements from NISAR.

Made minor alterations to the NISAR improvements.

Refs #16
---
 cluster_provisioning/setup_trigger_rules.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 cluster_provisioning/setup_trigger_rules.sh

diff --git a/cluster_provisioning/setup_trigger_rules.sh b/cluster_provisioning/setup_trigger_rules.sh
new file mode 100644
index 000000000..4264472f1
--- /dev/null
+++ b/cluster_provisioning/setup_trigger_rules.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+source $HOME/.bash_profile
+
+# fail on any errors
+set -ex
+
+curl -XDELETE http://$(curl http://169.254.169.254/latest/meta-data/local-ipv4):9200/user_rules-grq
+curl -XDELETE http://$(curl http://169.254.169.254/latest/meta-data/local-ipv4):9200/user_rules-mozart
+fab -f ~/.sds/cluster.py -R mozart,grq create_all_user_rules_index
+
+# import some tosca + figaro rules
+pushd ~/.sds/files/test
+./import_rules.sh
+./import_rules-mozart.sh
+./import_product_delivery_rules.sh
+popd

From 9f8e5d4910f432c555a4e39bb198bbe6f2601344 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 10 Mar 2022 13:49:52 -0800
Subject: [PATCH 27/48] feat(accountability): syncing some Terraform script
 improvements from NISAR.

Made minor alterations to the NISAR improvements.

Refs #16
---
 cluster_provisioning/modules/common/main.tf | 78 +++++++++++++++++++--
 1 file changed, 74 insertions(+), 4 deletions(-)

diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index b9aaaedd6..6254b5be7 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -1135,6 +1135,35 @@ resource "aws_instance" "mozart" {
     ]
   }
 
+  # To test HySDS core development (feature branches), uncomment this block
+  # and add lines to perform the mods to test them. Three examples have been
+  # left as described below:
+  #provisioner "remote-exec" {
+  #  inline = [
+  #    "set -ex",
+  #    "source ~/.bash_profile",
+
+  # Example 1: test a single file update from an sdscli feature branch named hotfix-sighup
+  #    "cd ~/mozart/ops/sdscli/sdscli/adapters/hysds",
+  #    "mv fabfile.py fabfile.py.bak",
+  #    "wget https://raw.githubusercontent.com/sdskit/sdscli/hotfix-sighup/sdscli/adapters/hysds/fabfile.py",
+
+  # Example 2: test an entire feature branch (need HYSDS_RELEASE=develop terraform variable)
+  #    "cd ~/mozart/ops/hysds",
+  #    "git checkout <dustins_branch>",
+  #    "pip install -e .",
+
+  # Example 3: test a custom verdi docker image on the ASGs (need HYSDS_RELEASE=develop terraform variable)
+  #    "cd ~/mozart/pkgs",
+  #    "mv hysds-verdi-develop.tar.gz hysds-verdi-develop.tar.gz.bak",
+  #    "docker pull hysds/verdi:<dustins_branch>",
+  #    "docker tag hysds/verdi:<dustins_branch> hysds/verdi:develop",
+  #    "docker save hysds/verdi:develop > hysds-verdi-develop.tar",
+  #    "pigz hysds-verdi-develop.tar",
+
+  #  ]
+  #}
+
   provisioner "remote-exec" {
     inline = [
       "set -ex",
@@ -1226,6 +1255,51 @@ resource "aws_instance" "mozart" {
   }
 }
 
+# Resource to install PCM and its dependencies
+resource "null_resource" "install_pcm_and_pges" {
+  depends_on = [
+    aws_instance.mozart
+  ]
+
+  connection {
+    type = "ssh"
+    host = aws_instance.mozart.private_ip
+    user = "hysdsops"
+    private_key = file(var.private_key_file)
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      "source ~/.bash_profile",
+      # build/import opera-pcm
+      "echo Build container",
+      "if [ \"${var.use_artifactory}\" = true ]; then",
+      "    ~/mozart/ops/opera-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/opera/sds/pcm/hysds_pkgs/container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    sds pkg import container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    rm -rf container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:\"container-iems-sds_opera-pcm:${lower(var.pcm_branch)}\"",
+      "else",
+      "    sds --debug ci add_job --branch ${var.pcm_branch} --token https://${var.pcm_repo} s3",
+      "    sds --debug ci build_job --branch ${var.pcm_branch} https://${var.pcm_repo}",
+      "    sds --debug ci remove_job --branch ${var.pcm_branch} https://${var.pcm_repo}",
+      "fi",
+      # build/import CNM product delivery
+      "if [ \"${var.use_artifactory}\" = true ]; then",
+      "    ~/mozart/ops/opera-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/opera/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
+      "    sds pkg import container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
+      "    rm -rf container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
+      "else",
+      "    sds --debug ci add_job --branch ${var.product_delivery_branch} --token https://${var.product_delivery_repo} s3",
+      "    sds --debug ci build_job --branch ${var.product_delivery_branch} https://${var.product_delivery_repo}",
+      "    sds --debug ci remove_job --branch ${var.product_delivery_branch} https://${var.product_delivery_repo}",
+      "fi",
+      "echo Set up trigger rules",
+      "sh ~/mozart/ops/opera-pcm/cluster_provisioning/setup_trigger_rules.sh"
+    ]
+  }
+}
+
 resource "null_resource" "destroy_es_snapshots" {
   triggers = {
     private_key_file   = var.private_key_file
@@ -1441,7 +1515,6 @@ resource "aws_autoscaling_policy" "autoscaling_policy" {
         name  = "Queue"
         value = each.key
       }
-#      metric_name = "JobsWaitingPerInstance-${var.project}-${var.venue}-${local.counter}-${each.key}"
       metric_name = "${lookup(each.value, "total_jobs_metric", false) ? "JobsPerInstance" : "JobsWaitingPerInstance"}-${var.project}-${var.venue}-${local.counter}-${each.key}"
       unit        = "None"
       namespace   = "HySDS"
@@ -1474,7 +1547,6 @@ resource "aws_instance" "metrics" {
   }
   #This is very important, as it tells terraform to not mess with tags
   lifecycle {
-#    ignore_changes = [tags]
     ignore_changes = [tags, volume_tags]
   }
   subnet_id              = var.subnet_id
@@ -1542,7 +1614,6 @@ resource "aws_instance" "grq" {
   }
   #This is very important, as it tells terraform to not mess with tags
   lifecycle {
-#    ignore_changes = [tags]
     ignore_changes = [tags, volume_tags]
   }
   subnet_id              = var.subnet_id
@@ -1620,7 +1691,6 @@ resource "aws_instance" "factotum" {
   }
   #This is very important, as it tells terraform to not mess with tags
   lifecycle {
-#    ignore_changes = [tags]
     ignore_changes = [tags, volume_tags]
   }
   subnet_id              = var.subnet_id

From dfb907bc5513c6cc7040df15c769a5ee34d6c6cb Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 24 Mar 2022 09:56:27 -0700
Subject: [PATCH 28/48] feat(accountability): PGE directories specified in job
 spec.

Refs #16
---
 docker/hysds-io.json.SCIFLO_L3_DSWx_HLS | 18 ++++++++++++++++++
 docker/job-spec.json.SCIFLO_L3_DSWx_HLS | 12 ++++++++++++
 opera_chimera/accountability.py         |  2 +-
 opera_chimera/postprocess_functions.py  |  3 ++-
 wrapper/opera_pge_wrapper.py            | 10 +++++-----
 5 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
index 851a04c35..ffe118347 100644
--- a/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/hysds-io.json.SCIFLO_L3_DSWx_HLS
@@ -47,6 +47,24 @@
       "from": "value",
       "type": "text",
       "value": "OperaAccountability"
+    },
+    {
+      "name": "pge_runconfig_dir",
+      "from": "value",
+      "type": "text",
+      "value": "pge_runconfig_dir"
+    },
+    {
+      "name": "pge_input_dir",
+      "from": "value",
+      "type": "text",
+      "value": "pge_input_dir"
+    },
+    {
+      "name": "pge_output_dir",
+      "from": "value",
+      "type": "text",
+      "value": "pge_output_dir"
     }
   ]
 }
diff --git a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
index 4cbfa3b94..de09e05e4 100644
--- a/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
+++ b/docker/job-spec.json.SCIFLO_L3_DSWx_HLS
@@ -52,6 +52,18 @@
     {
       "name": "accountability_class",
       "destination": "context"
+    },
+    {
+      "name": "pge_runconfig_dir",
+      "destination": "context"
+    },
+    {
+      "name": "pge_input_dir",
+      "destination": "context"
+    },
+    {
+      "name": "pge_output_dir",
+      "destination": "context"
     }
   ]
 }
diff --git a/opera_chimera/accountability.py b/opera_chimera/accountability.py
index ae4c4ff7d..16f272098 100644
--- a/opera_chimera/accountability.py
+++ b/opera_chimera/accountability.py
@@ -164,7 +164,7 @@ def flatten_and_merge_accountability(self):
     def update_product_met_json(self, job_result):
         """Creates a .met.json with updated accountability metadata."""
         work_dir = job_result.get("work_dir")
-        datasets_path = f"{work_dir}/output_dir_tbf/datasets/"  # TODO chrisjrd: resolve
+        datasets_path = f"{work_dir}/{self.context['pge_output_dir']}/datasets/"
         datasets = os.listdir(datasets_path)
         old_accountability = self.flatten_and_merge_accountability()
 
diff --git a/opera_chimera/postprocess_functions.py b/opera_chimera/postprocess_functions.py
index 83eedd617..f1ee471b0 100644
--- a/opera_chimera/postprocess_functions.py
+++ b/opera_chimera/postprocess_functions.py
@@ -3,6 +3,7 @@
 that are part of the OPERA PCM pipeline.
 
 """
+from typing import Dict
 
 from chimera.commons.constants import ChimeraConstants as chimera_consts
 from chimera.postprocess_functions import PostProcessFunctions
@@ -16,7 +17,7 @@
 
 
 class OperaPostProcessFunctions(PostProcessFunctions):
-    def __init__(self, context, pge_config, settings, job_result):
+    def __init__(self, context: Dict, pge_config: Dict, settings, job_result):
         PostProcessFunctions.__init__(
             self, context, pge_config, settings, job_result, mozart_es=mozart_es, grq_es=grq_es
         )
diff --git a/wrapper/opera_pge_wrapper.py b/wrapper/opera_pge_wrapper.py
index 13b26fc0f..ab674d0aa 100644
--- a/wrapper/opera_pge_wrapper.py
+++ b/wrapper/opera_pge_wrapper.py
@@ -43,7 +43,7 @@ def run_pipeline(context: Dict, work_dir: str) -> List[Union[bytes, str]]:
 
     logger.info(f"Preparing Working Directory: {work_dir}")
 
-    input_hls_dir, output_dir, runconfig_dir = create_required_directories(work_dir)
+    input_hls_dir, output_dir, runconfig_dir = create_required_directories(work_dir, context)
 
     run_config: Dict = context.get("run_config")
 
@@ -104,17 +104,17 @@ def run_pipeline(context: Dict, work_dir: str) -> List[Union[bytes, str]]:
     return created_datasets
 
 
-def create_required_directories(work_dir: str) -> Tuple[str, str, str]:
+def create_required_directories(work_dir: str, context: Dict) -> Tuple[str, str, str]:
     """Creates the requisite directories per PGE-PCM ICS for L3_DSWx_HLS."""
     logger.info("Creating directories for PGE.")
 
-    runconfig_dir = os.path.join(work_dir, 'runconfig_dir_tbf')
+    runconfig_dir = os.path.join(work_dir, context["pge_runconfig_dir"])
     os.makedirs(runconfig_dir, 0o755, exist_ok=True)
 
-    input_hls_dir = os.path.join(work_dir, 'input_hls_dir_tbf')
+    input_hls_dir = os.path.join(work_dir, context["pge_input_dir"])
     os.makedirs(input_hls_dir, 0o755, exist_ok=True)
 
-    output_dir = os.path.join(work_dir, 'output_dir_tbf')
+    output_dir = os.path.join(work_dir, context["pge_output_dir"])
     os.makedirs(output_dir, 0o755, exist_ok=True)
 
     return input_hls_dir, output_dir, runconfig_dir

From 6aba2cf63693b8093a69fd07a2f29d0579595173 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 24 Mar 2022 10:01:49 -0700
Subject: [PATCH 29/48] feat(accountability): adding log file to PGE output
 config.

Refs #16
---
 conf/pge_outputs.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/conf/pge_outputs.yaml b/conf/pge_outputs.yaml
index 6b4000f6f..db0c90a2a 100644
--- a/conf/pge_outputs.yaml
+++ b/conf/pge_outputs.yaml
@@ -1,3 +1,4 @@
+#######################################################################
 # This file defines the outputs that are expected to be produced by the
 # different PGEs within the PCM.
 # 
@@ -28,8 +29,8 @@
 #      - regex: !!python/regexp '^NISAR_[L|S|J]0_RRST_VC\w{2}_\d{8}T\d{6}_\d{8}T\d{6}_\w{6}_\d{3}\.bin\.met$'
 #    Optional: []
 #  Missing_Metadata: {}
+#######################################################################
 
-# TODO chrisjrd: Finalize config, adding regex for the other output files
 # Key name MUST match the corresponding RunConfig jinja2 template filename.
 L3_HLS:
   Outputs:
@@ -41,6 +42,7 @@ L3_HLS:
       - regex: !!python/regexp '(?P<id>(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)_(?P<source>HLS)_(?P<sensor>SENTINEL-2A|LANDSAT-8)_(?P<tile_id>T[^\W_]{5})_(?P<datetime>(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})T(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2}))_(?P<collection_version>v\d+[.]\d+)[.](?P<ext>tif))$'
         verify: true
         hash: md5
-    Secondary: []
+    Secondary:
+      - regex: !!python/regexp '(?P<id>(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)_(?P<source>HLS)_(?P<sensor>SENTINEL-2A|LANDSAT-8)_(?P<tile_id>T[^\W_]{5})_(?P<datetime>(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})T(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2}))_(?P<collection_version>v\d+[.]\d+)[.](?P<ext>log))$'
     Optional: []
   Missing_Metadata: {}

From 2d04e99d5d19708c365bd37d402f067d00fb396c Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Thu, 24 Mar 2022 12:57:35 -0700
Subject: [PATCH 30/48] feat(accountability): fix job spec param lookup in
 _context.json

Refs #16
---
 wrapper/opera_pge_wrapper.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/wrapper/opera_pge_wrapper.py b/wrapper/opera_pge_wrapper.py
index ab674d0aa..1df4f67fc 100644
--- a/wrapper/opera_pge_wrapper.py
+++ b/wrapper/opera_pge_wrapper.py
@@ -108,18 +108,30 @@ def create_required_directories(work_dir: str, context: Dict) -> Tuple[str, str,
     """Creates the requisite directories per PGE-PCM ICS for L3_DSWx_HLS."""
     logger.info("Creating directories for PGE.")
 
-    runconfig_dir = os.path.join(work_dir, context["pge_runconfig_dir"])
+    runconfig_dir = os.path.join(work_dir, job_param_by_name(context, "pge_runconfig_dir"))
     os.makedirs(runconfig_dir, 0o755, exist_ok=True)
 
-    input_hls_dir = os.path.join(work_dir, context["pge_input_dir"])
+    input_hls_dir = os.path.join(work_dir, job_param_by_name(context, "pge_input_dir"))
     os.makedirs(input_hls_dir, 0o755, exist_ok=True)
 
-    output_dir = os.path.join(work_dir, context["pge_output_dir"])
+    output_dir = os.path.join(work_dir, job_param_by_name(context, "pge_output_dir"))
     os.makedirs(output_dir, 0o755, exist_ok=True)
 
     return input_hls_dir, output_dir, runconfig_dir
 
 
+def job_param_by_name(context: Dict, name: str):
+    """
+    Gets the job specification parameter from the _context.json file.
+    :param context: the dict representation of _context.json.
+    :param name: the name of the job specification parameter.
+    """
+
+    for param in context["job_specification"]["params"]:
+        if param["name"] == name:
+            return param["value"]
+    raise Exception(f"param ({name}) not found in _context.json")
+
 def exec_pge_command(context: Dict, work_dir: str, input_hls_dir: str, runconfig_dir: str, output_dir: str):
     logger.info("Preparing PGE docker command.")
 

From d4d22ecb62bd3c3370895986cc0458c83c697a64 Mon Sep 17 00:00:00 2001
From: chrisjrd <chris.jrd@gmail.com>
Date: Fri, 25 Mar 2022 10:46:55 -0700
Subject: [PATCH 31/48] feat(accountability): remove unused constants

Refs #16
---
 commons/constants/product_metadata.py         |  1 -
 .../constants/opera_chimera_const.py          | 40 +------------------
 2 files changed, 1 insertion(+), 40 deletions(-)

diff --git a/commons/constants/product_metadata.py b/commons/constants/product_metadata.py
index c7a49aa0e..3e670a942 100644
--- a/commons/constants/product_metadata.py
+++ b/commons/constants/product_metadata.py
@@ -67,7 +67,6 @@
 
 # TODO: update these with what they will actually be
 OBS_ID = "OBS_ID"
-TOTAL_NUMBER_RANGELINES = "Total_Number_Rangelines"
 TOTAL_NUMBER_OF_MISSING_RANGELINES = "Total_Number_Of_Missing_Rangelines"
 TOTAL_NUMBER_OF_RANGELINES_FAILED_CHECKSUM = (
     "Total_Number_Of_Rangelines_Failed_Checksum"
diff --git a/opera_chimera/constants/opera_chimera_const.py b/opera_chimera/constants/opera_chimera_const.py
index 244e44264..3e7bac501 100644
--- a/opera_chimera/constants/opera_chimera_const.py
+++ b/opera_chimera/constants/opera_chimera_const.py
@@ -1,8 +1,5 @@
 from chimera.commons.constants import ChimeraConstants
 from job_accountability.catalog import ES_INDEX as job_accountability_index
-#from pass_accountability.catalog import ES_INDEX as pass_accountability_index
-#from observation_accountability.catalog import ES_INDEX as observation_accountability_index
-#from Track_Frame_Accountability.catalog import ES_INDEX as track_frame_accountability_index
 
 
 class OperaChimeraConstants(ChimeraConstants):
@@ -38,18 +35,12 @@ def __init__(self):
 
     SET_MOCK_METADATA = "set_mock_metadata"
 
-    SET_L0B_MOCK_METADATA = "set_l0b_mock_metadata"
-
     SET_BASE_NAME = "set_base_name"
 
-    SET_L0B_BASE_NAMES = "set_l0b_base_names"
-
     SET_EXTRA_PGE_OUTPUT_METADATA = "set_extra_pge_output_metadata"
 
     GET_OBSERVATIONS_DATA = "get_observations_data"
 
-    GET_L0B_ANCILLAY_FILES = "get_l0b_ancillary_files"
-
     GET_HARDCODED_METADATA = "get_hardcoded_metadata"
 
     SET_PCM_RETRIEVAL_ID = "set_pcm_retrieval_id"
@@ -84,23 +75,13 @@ def __init__(self):
     MOCK_METADATA = "mock_metadata"
     BASE_NAME = "base_name"
 
-    OROST = ""
-    LRCLK_FILE = "LRCLKFile"
     DEM_FILE = "DEMFile"
-    RADAR_CONFIGURATION_FILE = "RadarConfigurationFile"
-    CHIRP_PARAMETER_FILE = "ChirpParameterFile"
-    WAVE_CONFIGURATION_FILE = "WaveformConfigurationFile"
 
-    PLANNED_DATATAKE_ID = "PlannedDatatakeId"
     PLANNED_OBSERVATION_ID = "PlannedObservationId"
-    PLANNED_DATATAKE_TIMESTAMP = "PlannedDatatakeTimestamp"
     PLANNED_OBSERVATION_TIMESTAMP = "PlannedObservationTimestamp"
     IS_URGENT_OBSERVATION = "IsUrgentObservation"
     CONFIGURATION_ID = "ConfigurationId"
     MISSION_CYCLE = "MissionCycle"
-    TOTAL_NUMBER_RANGELINES = "TotalNumberRangelines"
-    RANGELINES_TO_SKIP = "RangelinesToSkip"
-    TOTAL_RANGELINES_FAILED_CHECKSUM = "TotalNumberOfRangelinesFailedChecksum"
     START_TIME = "StartTime"
     END_TIME = "EndTime"
     IS_URGENT = "is_urgent"
@@ -112,13 +93,8 @@ def __init__(self):
     ORBIT_DIRECTION = "OrbitDirection"
     LOOK_DIRECTION = "LookDirection"
 
-    OROST = "orost"
-    SROST = "srost"
-    OFS = "ofs"
 
-    CHIRP_PARAM = "chirp_param"
     SCLKSCET = "sclkscet"
-    RADAR_CFG = "radar_cfg"
 
     REFINED_POINTING = "Pointing"
     ANT_PATTERN = "AntennaPattern"
@@ -140,21 +116,10 @@ def __init__(self):
     GPU_ENABLED = "gpu_enabled"
 
     # PGE names
-    #L0A = "L0A"
-    #TIME_EXTRACTOR = "Time_Extractor"
-    #L0B = "L0B"
+    #L3_DSWX_HLS = "L3_DSWX_HLS"
 
     # Other Constants
     JOB_ACCOUNTABILITY_INDEX = job_accountability_index
-    #PASS_ACCOUNTABILITY_INDEX = pass_accountability_index
-    #OBSERVATION_ACCOUNTABILITY_INDEX = observation_accountability_index
-    #TRACK_FRAME_ACCOUNTABILITY_INDEX = track_frame_accountability_index
-
-    # state config dataset types
-    #DATATAKE_STATE_CONFIG_DOC_TYPE = "datatake-state-config"
-    #DATATAKE_EXP_STATE_CONFIG_DOC_TYPE = "datatake-expired-state-config"
-    #DATATAKE_UR_EXP_STATE_CONFIG_DOC_TYPE = "datatake-urgent_response_expired-state-config"
-    #DATATAKE_UR_STATE_CONFIG_DOC_TYPE = "datatake-urgent_response_state-config"
 
     TASK_INDEX = "task_status-current"
     TASK_ID_FIELD = "task_id"
@@ -167,9 +132,6 @@ def __init__(self):
 
     EXTRA_PGE_OUTPUT_METADATA = "extra_pge_output_metadata"
 
-    # job-specific parameter extraction/coercion from context
-    #GET_GCOV_JOB_PARAMS_FROM_CONTEXT = "get_gcov_job_params_from_context"
-
     GET_PRODUCTS = "get_products"
     IS_STATE_CONFIG_TRIGGER = "is_state_config_trigger"
     FILE_NAMES_KEY = "file_names_key"

From 68af47487437b826302af869af40f8f01109e364 Mon Sep 17 00:00:00 2001
From: Maxwell Plotkin <maxwell.a.plotkin@jpl.nasa.gov>
Date: Wed, 30 Mar 2022 13:33:53 -0700
Subject: [PATCH 32/48] Add index_datetime field to data subscriber index

---
 data_subscriber/catalog.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/data_subscriber/catalog.py b/data_subscriber/catalog.py
index aac788dda..37fa6e7b0 100644
--- a/data_subscriber/catalog.py
+++ b/data_subscriber/catalog.py
@@ -31,7 +31,9 @@ def delete_index(self, index=ES_INDEX):
             self.logger.info("Successfully deleted index: {}".format(index))
 
     def post(self, id, url, index=ES_INDEX):
-        result = self.index_document(index=index, body={"url": url, "downloaded": False}, id=id)
+        result = self.index_document(index=index,
+                                     body={"url": url, "downloaded": False, "index_datetime": str(datetime.now())},
+                                     id=id)
 
         if self.logger:
             self.logger.debug(f"Document indexed: {result}")
@@ -39,14 +41,12 @@ def post(self, id, url, index=ES_INDEX):
     def mark_downloaded(self, id, index=ES_INDEX):
         result = self.update_document(id=id,
                                       body={"doc_as_upsert": True,
-                                            "doc": {"downloaded": True, "download_date": str(datetime.now())}},
+                                            "doc": {"downloaded": True, "download_datetime": str(datetime.now())}},
                                       index=index)
 
         if self.logger:
             self.logger.debug(f"Document updated: {result}")
 
-        # return result
-
     def query_existence(self, id, index=ES_INDEX):
         try:
             result = self.get_by_id(index=index, id=id)
@@ -62,7 +62,8 @@ def query_existence(self, id, index=ES_INDEX):
 
     def query_undownloaded(self, index=ES_INDEX):
         try:
-            result = self.query(index=index, body={"query": {"match": {"downloaded": False}}})
+            result = self.query(index=index,
+                                body={"sort": [{"index_datetime": "asc"}], "query": {"match": {"downloaded": False}}})
             if self.logger:
                 self.logger.debug(f"Query result: {result}")
 

From 60e7771431932b3c3f000b707bda7dce600d7b07 Mon Sep 17 00:00:00 2001
From: Maxwell Plotkin <maxwell.a.plotkin@jpl.nasa.gov>
Date: Wed, 30 Mar 2022 14:25:23 -0700
Subject: [PATCH 33/48] Issue 67 logging improvements

---
 data_subscriber/daac_data_subscriber.py          | 8 ++++----
 data_subscriber/daac_data_subscriber_download.sh | 4 ++--
 data_subscriber/daac_data_subscriber_query.sh    | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/data_subscriber/daac_data_subscriber.py b/data_subscriber/daac_data_subscriber.py
index b4f2e9b0c..6464f6895 100755
--- a/data_subscriber/daac_data_subscriber.py
+++ b/data_subscriber/daac_data_subscriber.py
@@ -252,10 +252,10 @@ def query_cmr(args, token, CMR):
         temporal_range = get_temporal_range(args.startDate, args.endDate,
                                             datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
         params['temporal'] = temporal_range
-        logging.debug("Temporal Range: " + temporal_range)
+        logging.info("Temporal Range: " + temporal_range)
 
-    logging.debug("Provider: " + args.provider)
-    logging.debug("Updated Since: " + data_within_last_timestamp)
+    logging.info("Provider: " + args.provider)
+    logging.info("Updated Since: " + data_within_last_timestamp)
 
     product_urls, search_after = request_search(url, params)
 
@@ -269,7 +269,7 @@ def query_cmr(args, token, CMR):
                      for extension in EXTENSION_LIST_MAP.get(args.extension_list.upper())
                      if extension in f]
 
-    logging.debug(f"Found {str(len(filtered_urls))} total files")
+    logging.info(f"Found {str(len(filtered_urls))} total files")
 
     return filtered_urls
 
diff --git a/data_subscriber/daac_data_subscriber_download.sh b/data_subscriber/daac_data_subscriber_download.sh
index 4fc759a86..a99cdc542 100755
--- a/data_subscriber/daac_data_subscriber_download.sh
+++ b/data_subscriber/daac_data_subscriber_download.sh
@@ -21,8 +21,8 @@ echo "Running job to download data"
 date
 
 # Forward processing use case; download all undownloaded files from ES index
-echo "python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download --verbose"
-python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download --verbose
+echo "python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download"
+python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download 2>&1
 STATUS=$?
 
 if [ $STATUS -eq 0 ]; then
diff --git a/data_subscriber/daac_data_subscriber_query.sh b/data_subscriber/daac_data_subscriber_query.sh
index 4e82dcae6..e564ea83f 100755
--- a/data_subscriber/daac_data_subscriber_query.sh
+++ b/data_subscriber/daac_data_subscriber_query.sh
@@ -21,12 +21,12 @@ echo "Running job to query LPDAAC data"
 date
 
 # Forward processing use case; query previous 60 minutes
-echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query --verbose"
-python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query --verbose
+echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query"
+python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query 2>&1
 STATUS_L30=$?
 
-echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query --verbose"
-python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query --verbose
+echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query"
+python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query 2>&1
 STATUS_S30=$?
 
 if [ $STATUS_S30 -eq 0 ] && [ $STATUS_L30 -eq 0 ]; then

From 49846bcae7a29b389233d99192f3b26d2b1dd9ed Mon Sep 17 00:00:00 2001
From: Maxwell Plotkin <maxwell.a.plotkin@jpl.nasa.gov>
Date: Wed, 30 Mar 2022 15:09:34 -0700
Subject: [PATCH 34/48] Explicitly define index fields for data subscriber ES

---
 data_subscriber/catalog.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/data_subscriber/catalog.py b/data_subscriber/catalog.py
index 37fa6e7b0..0c52df7f0 100644
--- a/data_subscriber/catalog.py
+++ b/data_subscriber/catalog.py
@@ -21,7 +21,14 @@ class DataSubscriberProductCatalog(ElasticsearchUtility):
     """
 
     def create_index(self, index=ES_INDEX):
-        self.es.indices.create(index=ES_INDEX)
+        self.es.indices.create(body={"settings": {"index": {"sort.field": "index_datetime", "sort.order": "asc"}},
+                                     "mappings": {
+                                         "properties": {
+                                             "url": {"type": "keyword"},
+                                             "index_datetime": {"type": "date"},
+                                             "download_datetime": {"type": "date"},
+                                             "downloaded": {"type": "boolean"}}}},
+                               index=ES_INDEX)
         if self.logger:
             self.logger.info("Successfully created index: {}".format(index))
 
@@ -32,7 +39,7 @@ def delete_index(self, index=ES_INDEX):
 
     def post(self, id, url, index=ES_INDEX):
         result = self.index_document(index=index,
-                                     body={"url": url, "downloaded": False, "index_datetime": str(datetime.now())},
+                                     body={"url": url, "downloaded": False, "index_datetime": datetime.now()},
                                      id=id)
 
         if self.logger:
@@ -41,7 +48,7 @@ def post(self, id, url, index=ES_INDEX):
     def mark_downloaded(self, id, index=ES_INDEX):
         result = self.update_document(id=id,
                                       body={"doc_as_upsert": True,
-                                            "doc": {"downloaded": True, "download_datetime": str(datetime.now())}},
+                                            "doc": {"downloaded": True, "download_datetime": datetime.now()}},
                                       index=index)
 
         if self.logger:

From 4136a3e8cbce621e02de8a52617b62b6b970bb2b Mon Sep 17 00:00:00 2001
From: Maxwell Plotkin <maxwell.a.plotkin@jpl.nasa.gov>
Date: Wed, 30 Mar 2022 15:48:07 -0700
Subject: [PATCH 35/48] Remove temporary file after s3 upload to save storage

---
 data_subscriber/daac_data_subscriber.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/data_subscriber/daac_data_subscriber.py b/data_subscriber/daac_data_subscriber.py
index 6464f6895..22607b059 100755
--- a/data_subscriber/daac_data_subscriber.py
+++ b/data_subscriber/daac_data_subscriber.py
@@ -397,10 +397,12 @@ def s3_transfer(url, bucket_name, s3, tmp_dir, staging_area=""):
     target_key = os.path.join(staging_area, file_name)
 
     try:
-        s3.download_file(source_bucket, source_key, f"{tmp_dir}/{target_key}")
+        tmp_path = f"{tmp_dir}/{target_key}"
+        s3.download_file(source_bucket, source_key, tmp_path)
 
         target_s3 = boto3.resource("s3")
-        target_s3.Bucket(target_bucket).upload_file(f"{tmp_dir}/{target_key}", target_key)
+        target_s3.Bucket(target_bucket).upload_file(tmp_path, target_key)
+        os.remove(tmp_path)
 
         return {"successful_download": target_key}
     except Exception as e:

From 7c73d91f6cfeba4396954488239d6be9572e9fc7 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Wed, 30 Mar 2022 23:50:11 -0700
Subject: [PATCH 36/48] Fix merge error

---
 .../dev-e2e-baseline-pge/main.tf              |  3 +-
 .../dev-e2e-baseline-pge/variables.tf         |  8 +++--
 .../dev-e2e-event-misfire/main.tf             |  3 +-
 .../dev-e2e-event-misfire/variables.tf        |  8 +++--
 cluster_provisioning/dev-e2e-pge/main.tf      |  5 +--
 cluster_provisioning/dev-e2e-pge/variables.tf |  8 +++--
 cluster_provisioning/dev-e2e/main.tf          |  5 ++-
 cluster_provisioning/dev-e2e/variables.tf     |  4 +--
 cluster_provisioning/dev-int/main.tf          |  5 +--
 cluster_provisioning/dev-releaser/main.tf     |  3 +-
 .../dev-releaser/variables.tf                 |  6 +++-
 .../dev-restore-snapshot/main.tf              |  3 +-
 .../dev-restore-snapshot/variables.tf         |  6 +++-
 cluster_provisioning/dev/main.tf              |  4 +--
 cluster_provisioning/dev/variables.tf         |  6 ++--
 cluster_provisioning/int/variables.tf         |  8 +++--
 cluster_provisioning/modules/common/main.tf   | 33 +++++++------------
 .../modules/common/variables.tf               |  4 +--
 18 files changed, 70 insertions(+), 52 deletions(-)

diff --git a/cluster_provisioning/dev-e2e-baseline-pge/main.tf b/cluster_provisioning/dev-e2e-baseline-pge/main.tf
index cf1b3f8b5..afaa199b6 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/main.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
index 21215b6a1..9b089f442 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
@@ -369,7 +369,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -429,4 +433,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e-event-misfire/main.tf b/cluster_provisioning/dev-e2e-event-misfire/main.tf
index f9d567986..69ccb17a6 100644
--- a/cluster_provisioning/dev-e2e-event-misfire/main.tf
+++ b/cluster_provisioning/dev-e2e-event-misfire/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-e2e-event-misfire/variables.tf b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
index d3be83945..54cf0d55b 100644
--- a/cluster_provisioning/dev-e2e-event-misfire/variables.tf
+++ b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
@@ -361,7 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -421,4 +425,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e-pge/main.tf b/cluster_provisioning/dev-e2e-pge/main.tf
index 640c74a8f..65221d0cd 100644
--- a/cluster_provisioning/dev-e2e-pge/main.tf
+++ b/cluster_provisioning/dev-e2e-pge/main.tf
@@ -75,8 +75,9 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
-  #obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
+  obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
   code_bucket                             = var.code_bucket
diff --git a/cluster_provisioning/dev-e2e-pge/variables.tf b/cluster_provisioning/dev-e2e-pge/variables.tf
index 82fb2e85e..a2fcc0a1d 100644
--- a/cluster_provisioning/dev-e2e-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-pge/variables.tf
@@ -407,7 +407,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -471,4 +475,4 @@ variable "earthdata_pass" {
 
 variable "pge_test_package" {
   default = "testdata_R1.0.0"
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e/main.tf b/cluster_provisioning/dev-e2e/main.tf
index aa027b001..c488498b6 100644
--- a/cluster_provisioning/dev-e2e/main.tf
+++ b/cluster_provisioning/dev-e2e/main.tf
@@ -73,7 +73,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
@@ -93,8 +94,6 @@ module "common" {
   es_snapshot_bucket                      = var.es_snapshot_bucket
   es_bucket_role_arn                      = var.es_bucket_role_arn
   run_smoke_test                          = var.run_smoke_test
-  data_subscriber_download_timer_trigger_frequency = ""
-  data_subscriber_query_timer_trigger_frequency = ""
 }
 
 locals {
diff --git a/cluster_provisioning/dev-e2e/variables.tf b/cluster_provisioning/dev-e2e/variables.tf
index 5ea05610f..48e8dc7fd 100644
--- a/cluster_provisioning/dev-e2e/variables.tf
+++ b/cluster_provisioning/dev-e2e/variables.tf
@@ -361,11 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_download_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
-variable "data_subscriber_query_timer_trigger_frequency" {
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev-int/main.tf b/cluster_provisioning/dev-int/main.tf
index f3409a91b..ceecf60a4 100644
--- a/cluster_provisioning/dev-int/main.tf
+++ b/cluster_provisioning/dev-int/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
@@ -151,7 +152,7 @@ resource "null_resource" "mozart" {
       "  ${var.use_daac_cnm} \\",
       "  ${local.crid} \\",
       "  ${var.cluster_type}  || :",
-#      "  \"${var.data_subscriber_timer_trigger_frequency}\" || :",
+#      "  \"${var.data_download_timer_trigger_frequency}\" || :",
       "fi",
     ]
   }
diff --git a/cluster_provisioning/dev-releaser/main.tf b/cluster_provisioning/dev-releaser/main.tf
index f7b2cc640..973784926 100644
--- a/cluster_provisioning/dev-releaser/main.tf
+++ b/cluster_provisioning/dev-releaser/main.tf
@@ -71,7 +71,8 @@ module "common" {
   use_daac_cnm                            = var.use_daac_cnm
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-releaser/variables.tf b/cluster_provisioning/dev-releaser/variables.tf
index 34ab32eec..77670759b 100644
--- a/cluster_provisioning/dev-releaser/variables.tf
+++ b/cluster_provisioning/dev-releaser/variables.tf
@@ -346,7 +346,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev-restore-snapshot/main.tf b/cluster_provisioning/dev-restore-snapshot/main.tf
index 43818bee1..5f2130a0c 100644
--- a/cluster_provisioning/dev-restore-snapshot/main.tf
+++ b/cluster_provisioning/dev-restore-snapshot/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-restore-snapshot/variables.tf b/cluster_provisioning/dev-restore-snapshot/variables.tf
index e3859dac5..1eef21a04 100644
--- a/cluster_provisioning/dev-restore-snapshot/variables.tf
+++ b/cluster_provisioning/dev-restore-snapshot/variables.tf
@@ -361,7 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev/main.tf b/cluster_provisioning/dev/main.tf
index aa810f384..96966b9e7 100644
--- a/cluster_provisioning/dev/main.tf
+++ b/cluster_provisioning/dev/main.tf
@@ -73,8 +73,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_download_timer_trigger_frequency = var.data_subscriber_download_timer_trigger_frequency
-  data_subscriber_query_timer_trigger_frequency = var.data_subscriber_query_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev/variables.tf b/cluster_provisioning/dev/variables.tf
index 3c0470507..fcc4ae97f 100644
--- a/cluster_provisioning/dev/variables.tf
+++ b/cluster_provisioning/dev/variables.tf
@@ -361,11 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_download_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
-variable "data_subscriber_query_timer_trigger_frequency" {
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -433,4 +433,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/int/variables.tf b/cluster_provisioning/int/variables.tf
index 2f42e62c3..14cd15f6e 100644
--- a/cluster_provisioning/int/variables.tf
+++ b/cluster_provisioning/int/variables.tf
@@ -330,7 +330,11 @@ variable "cluster_type" {
   default = ""
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -455,4 +459,4 @@ variable "queues" {
       "total_jobs_metric" = true
     }
   }
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 9e4c02c4e..cc4306faf 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -27,8 +27,8 @@ locals {
   maturity                          = split("-", var.daac_delivery_proxy)[5]
   timer_handler_job_type            = "timer_handler"
   accountability_report_job_type    = "accountability_report"
-  data_subscriber_download_job_type = "data_subscriber_download"
-  data_subscriber_query_job_type    = "data_subscriber_query"
+  data_download_job_type            = "data_download"
+  data_query_job_type               = "data_query"
   use_s3_uri_structure              = var.use_s3_uri_structure
   grq_es_url                        = "${var.grq_aws_es ? "https" : "http"}://${var.grq_aws_es ? var.grq_aws_es_host : aws_instance.grq.private_ip}:${var.grq_aws_es ? var.grq_aws_es_port : 9200}"
 
@@ -73,9 +73,6 @@ resource "null_resource" "download_lambdas" {
   provisioner "local-exec" {
     command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data-subscriber-query_handler_package_name}-${var.lambda_package_release}.zip"
   }
-#  provisioner "local-exec" {
-#    command = "curl ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data_subscriber_download_handler_package_name}-${var.lambda_package_release}.zip -o ${var.lambda_data_subscriber_download_handler_package_name}-${var.lambda_package_release}.zip"
-#  }
 }
 
 resource "null_resource" "is_cnm_r_event_trigger_value_valid" {
@@ -1006,7 +1003,6 @@ resource "aws_instance" "mozart" {
       "echo >> ~/.sds/config",
 
       "echo GIT_OAUTH_TOKEN: ${var.git_auth_key} >> ~/.sds/config",
-#	  "echo PUT_GIT_OAUTH_TOKEN: ${var.pub_git_auth_key} >> ~/.sds/config",
       "echo >> ~/.sds/config",
 
       "echo PROVES_URL: https://prov-es.jpl.nasa.gov/beta >> ~/.sds/config",
@@ -1074,7 +1070,6 @@ resource "aws_instance" "mozart" {
       "  ./install.sh mozart -d",
       "  rm -rf ~/mozart/pkgs/hysds-verdi-latest.tar.gz",
       "else",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
@@ -1091,10 +1086,10 @@ resource "aws_instance" "mozart" {
       "fi",
       "cd ~/mozart/ops",
       "if [ \"${var.use_artifactory}\" = true ]; then",
-      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-pcm-${var.pcm_branch}.tar.gz\"",
-      "  tar xfz ${var.project}-sds-pcm-${var.pcm_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-pcm-${var.pcm_branch} /export/home/hysdsops/mozart/ops/${var.project}-pcm",
-      "  rm -rf ${var.project}-sds-pcm-${var.pcm_branch}.tar.gz ",
+      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-pcm-${var.pcm_branch}.tar.gz\"",
+      "  tar xfz ${var.project}-pcm-${var.pcm_branch}.tar.gz",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-pcm-${var.pcm_branch} /export/home/hysdsops/mozart/ops/${var.project}-pcm",
+      "  rm -rf ${var.project}-pcm-${var.pcm_branch}.tar.gz ",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/CNM_product_delivery-${var.product_delivery_branch}.tar.gz\"",
       "  tar xfz CNM_product_delivery-${var.product_delivery_branch}.tar.gz",
       "  ln -s /export/home/hysdsops/mozart/ops/CNM_product_delivery-${var.product_delivery_branch} /export/home/hysdsops/mozart/ops/CNM_product_delivery",
@@ -1105,7 +1100,7 @@ resource "aws_instance" "mozart" {
       "  rm -rf pcm_commons-${var.pcm_commons_branch}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/bach-api",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/bach-api",
       "  rm -rf ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz ",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz",
@@ -1557,14 +1552,12 @@ resource "aws_instance" "metrics" {
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "fi"
@@ -1625,14 +1618,12 @@ resource "aws_instance" "grq" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-grq_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-grq_venv-${var.hysds_release}.tar.gz",
@@ -1715,14 +1706,12 @@ resource "aws_instance" "factotum" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#     "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-verdi_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-verdi_venv-${var.hysds_release}.tar.gz",
@@ -1986,7 +1975,7 @@ resource "aws_lambda_function" "data_subscriber_download_timer" {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
       "JOB_QUEUE": "opera-job_worker-small",
-      "JOB_TYPE": local.data_subscriber_download_job_type,
+      "JOB_TYPE": local.data_download_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
       "ISL_STAGING_AREA": var.isl_staging_area,
@@ -2006,7 +1995,7 @@ resource "aws_cloudwatch_log_group" "data_subscriber_download_timer" {
 resource "aws_cloudwatch_event_rule" "data_subscriber_download_timer" {
   name = "${aws_lambda_function.data_subscriber_download_timer.function_name}-Trigger"
   description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
-  schedule_expression = var.data_subscriber_download_timer_trigger_frequency
+  schedule_expression = var.data_download_timer_trigger_frequency
   is_enabled = local.enable_timer
 }
 
@@ -2041,7 +2030,7 @@ resource "aws_lambda_function" "data_subscriber_query_timer" {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
       "JOB_QUEUE": "factotum-job_worker-small",
-      "JOB_TYPE": local.data_subscriber_query_job_type,
+      "JOB_TYPE": local.data_query_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
       "ISL_STAGING_AREA": var.isl_staging_area,
@@ -2060,7 +2049,7 @@ resource "aws_cloudwatch_log_group" "data_subscriber_query_timer" {
 resource "aws_cloudwatch_event_rule" "data_subscriber_query_timer" {
   name = "${aws_lambda_function.data_subscriber_query_timer.function_name}-Trigger"
   description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
-  schedule_expression = var.data_subscriber_query_timer_trigger_frequency
+  schedule_expression = var.data_query_timer_trigger_frequency
   is_enabled = local.enable_timer
 }
 
diff --git a/cluster_provisioning/modules/common/variables.tf b/cluster_provisioning/modules/common/variables.tf
index 363d0e3e5..fc26545a5 100644
--- a/cluster_provisioning/modules/common/variables.tf
+++ b/cluster_provisioning/modules/common/variables.tf
@@ -352,9 +352,9 @@ variable "crid" {
 
 variable "obs_acct_report_timer_trigger_frequency" {}
 
-variable "data_subscriber_download_timer_trigger_frequency" {}
+variable "data_download_timer_trigger_frequency" {}
 
-variable "data_subscriber_query_timer_trigger_frequency" {}
+variable "data_query_timer_trigger_frequency" {}
 
 variable "cluster_type" {}
 

From d773ee51e87ce4334d474659ba1210be9aced776 Mon Sep 17 00:00:00 2001
From: Hyun Lee <47543396+hhlee445@users.noreply.github.com>
Date: Fri, 1 Apr 2022 12:38:43 -0700
Subject: [PATCH 37/48] Revert "Issues 66 and 67"

---
 data_subscriber/catalog.py                    | 20 ++++++-------------
 data_subscriber/daac_data_subscriber.py       | 14 ++++++-------
 .../daac_data_subscriber_download.sh          |  4 ++--
 data_subscriber/daac_data_subscriber_query.sh |  8 ++++----
 4 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/data_subscriber/catalog.py b/data_subscriber/catalog.py
index 0c52df7f0..aac788dda 100644
--- a/data_subscriber/catalog.py
+++ b/data_subscriber/catalog.py
@@ -21,14 +21,7 @@ class DataSubscriberProductCatalog(ElasticsearchUtility):
     """
 
     def create_index(self, index=ES_INDEX):
-        self.es.indices.create(body={"settings": {"index": {"sort.field": "index_datetime", "sort.order": "asc"}},
-                                     "mappings": {
-                                         "properties": {
-                                             "url": {"type": "keyword"},
-                                             "index_datetime": {"type": "date"},
-                                             "download_datetime": {"type": "date"},
-                                             "downloaded": {"type": "boolean"}}}},
-                               index=ES_INDEX)
+        self.es.indices.create(index=ES_INDEX)
         if self.logger:
             self.logger.info("Successfully created index: {}".format(index))
 
@@ -38,9 +31,7 @@ def delete_index(self, index=ES_INDEX):
             self.logger.info("Successfully deleted index: {}".format(index))
 
     def post(self, id, url, index=ES_INDEX):
-        result = self.index_document(index=index,
-                                     body={"url": url, "downloaded": False, "index_datetime": datetime.now()},
-                                     id=id)
+        result = self.index_document(index=index, body={"url": url, "downloaded": False}, id=id)
 
         if self.logger:
             self.logger.debug(f"Document indexed: {result}")
@@ -48,12 +39,14 @@ def post(self, id, url, index=ES_INDEX):
     def mark_downloaded(self, id, index=ES_INDEX):
         result = self.update_document(id=id,
                                       body={"doc_as_upsert": True,
-                                            "doc": {"downloaded": True, "download_datetime": datetime.now()}},
+                                            "doc": {"downloaded": True, "download_date": str(datetime.now())}},
                                       index=index)
 
         if self.logger:
             self.logger.debug(f"Document updated: {result}")
 
+        # return result
+
     def query_existence(self, id, index=ES_INDEX):
         try:
             result = self.get_by_id(index=index, id=id)
@@ -69,8 +62,7 @@ def query_existence(self, id, index=ES_INDEX):
 
     def query_undownloaded(self, index=ES_INDEX):
         try:
-            result = self.query(index=index,
-                                body={"sort": [{"index_datetime": "asc"}], "query": {"match": {"downloaded": False}}})
+            result = self.query(index=index, body={"query": {"match": {"downloaded": False}}})
             if self.logger:
                 self.logger.debug(f"Query result: {result}")
 
diff --git a/data_subscriber/daac_data_subscriber.py b/data_subscriber/daac_data_subscriber.py
index 22607b059..b4f2e9b0c 100755
--- a/data_subscriber/daac_data_subscriber.py
+++ b/data_subscriber/daac_data_subscriber.py
@@ -252,10 +252,10 @@ def query_cmr(args, token, CMR):
         temporal_range = get_temporal_range(args.startDate, args.endDate,
                                             datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
         params['temporal'] = temporal_range
-        logging.info("Temporal Range: " + temporal_range)
+        logging.debug("Temporal Range: " + temporal_range)
 
-    logging.info("Provider: " + args.provider)
-    logging.info("Updated Since: " + data_within_last_timestamp)
+    logging.debug("Provider: " + args.provider)
+    logging.debug("Updated Since: " + data_within_last_timestamp)
 
     product_urls, search_after = request_search(url, params)
 
@@ -269,7 +269,7 @@ def query_cmr(args, token, CMR):
                      for extension in EXTENSION_LIST_MAP.get(args.extension_list.upper())
                      if extension in f]
 
-    logging.info(f"Found {str(len(filtered_urls))} total files")
+    logging.debug(f"Found {str(len(filtered_urls))} total files")
 
     return filtered_urls
 
@@ -397,12 +397,10 @@ def s3_transfer(url, bucket_name, s3, tmp_dir, staging_area=""):
     target_key = os.path.join(staging_area, file_name)
 
     try:
-        tmp_path = f"{tmp_dir}/{target_key}"
-        s3.download_file(source_bucket, source_key, tmp_path)
+        s3.download_file(source_bucket, source_key, f"{tmp_dir}/{target_key}")
 
         target_s3 = boto3.resource("s3")
-        target_s3.Bucket(target_bucket).upload_file(tmp_path, target_key)
-        os.remove(tmp_path)
+        target_s3.Bucket(target_bucket).upload_file(f"{tmp_dir}/{target_key}", target_key)
 
         return {"successful_download": target_key}
     except Exception as e:
diff --git a/data_subscriber/daac_data_subscriber_download.sh b/data_subscriber/daac_data_subscriber_download.sh
index a99cdc542..4fc759a86 100755
--- a/data_subscriber/daac_data_subscriber_download.sh
+++ b/data_subscriber/daac_data_subscriber_download.sh
@@ -21,8 +21,8 @@ echo "Running job to download data"
 date
 
 # Forward processing use case; download all undownloaded files from ES index
-echo "python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download"
-python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download 2>&1
+echo "python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download --verbose"
+python $BASE_PATH/daac_data_subscriber.py -c ALL -s $ISL_BUCKET_NAME --index-mode download --verbose
 STATUS=$?
 
 if [ $STATUS -eq 0 ]; then
diff --git a/data_subscriber/daac_data_subscriber_query.sh b/data_subscriber/daac_data_subscriber_query.sh
index e564ea83f..4e82dcae6 100755
--- a/data_subscriber/daac_data_subscriber_query.sh
+++ b/data_subscriber/daac_data_subscriber_query.sh
@@ -21,12 +21,12 @@ echo "Running job to query LPDAAC data"
 date
 
 # Forward processing use case; query previous 60 minutes
-echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query"
-python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query 2>&1
+echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query --verbose"
+python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSL30 -s $ISL_BUCKET_NAME -e L30 --index-mode query --verbose
 STATUS_L30=$?
 
-echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query"
-python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query 2>&1
+echo "python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query --verbose"
+python $BASE_PATH/daac_data_subscriber.py -m 60 -c HLSS30 -s $ISL_BUCKET_NAME -e S30 --index-mode query --verbose
 STATUS_S30=$?
 
 if [ $STATUS_S30 -eq 0 ] && [ $STATUS_L30 -eq 0 ]; then

From 26ee2beccbbc75784e58699b3c4b0b331313a669 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Fri, 1 Apr 2022 13:10:18 -0700
Subject: [PATCH 38/48] Modify user_rules-cnm.json.tmpl due to change for
 CamelCase -> snake_case migration

---
 conf/sds/rules/user_rules-cnm.json.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/conf/sds/rules/user_rules-cnm.json.tmpl b/conf/sds/rules/user_rules-cnm.json.tmpl
index b4754d241..49b645e7a 100644
--- a/conf/sds/rules/user_rules-cnm.json.tmpl
+++ b/conf/sds/rules/user_rules-cnm.json.tmpl
@@ -3,7 +3,7 @@
     {
       "enabled": true,
       "job_type": "hysds-io-send_notify_msg:__TAG__",
-      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ DAAC_PROXY }}\",\"kinesis_role_arn\":\"\",\"sqs_queue_url\":\"{{ DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*.tif\"],\"metadata\":[\"*.context.json\",\"*.met.json\",\"*.dataset.json\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\"}",
+      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ DAAC_PROXY }}\",\"kinesis_role_arn\":\"\",\"sqs_queue_url\":\"{{ DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*.tif\"],\"metadata\":[\"*.context.json\",\"*.met.json\",\"*.dataset.json\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\",\"product_type_key\": \"ProductType\"}",
       "passthru_query": false,
       "priority": 0,
       "query_all": false,
@@ -17,7 +17,7 @@
     {
       "enabled": true,
       "job_type": "hysds-io-send_notify_msg:__TAG__",
-      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ DAAC_PROXY }}\",\"kinesis_role_arn\":\"\",\"sqs_queue_url\":\"{{ DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*.xml\"],\"metadata\":[\"*.context.json\",\"*.met.json\",\"*.dataset.json\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\"}",
+      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ DAAC_PROXY }}\",\"kinesis_role_arn\":\"\",\"sqs_queue_url\":\"{{ DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*.xml\"],\"metadata\":[\"*.context.json\",\"*.met.json\",\"*.dataset.json\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\",\"product_type_key\": \"ProductType\"}",
       "passthru_query": false,
       "priority": 0,
       "query_all": false,

From d4e62d1876c11b3450b63e332db9728baf6cff07 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Fri, 1 Apr 2022 20:55:51 -0700
Subject: [PATCH 39/48] Clean up main.tf and revert back data_download_job_type
 and data_download_job_type values

---
 cluster_provisioning/modules/common/main.tf | 39 ++++++++-------------
 1 file changed, 15 insertions(+), 24 deletions(-)

diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 9e4c02c4e..4595b2af3 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -27,8 +27,8 @@ locals {
   maturity                          = split("-", var.daac_delivery_proxy)[5]
   timer_handler_job_type            = "timer_handler"
   accountability_report_job_type    = "accountability_report"
-  data_subscriber_download_job_type = "data_subscriber_download"
-  data_subscriber_query_job_type    = "data_subscriber_query"
+  data_download_job_type            = "data_subscriber_download"
+  data_query_job_type               = "data_subscriber_query" 
   use_s3_uri_structure              = var.use_s3_uri_structure
   grq_es_url                        = "${var.grq_aws_es ? "https" : "http"}://${var.grq_aws_es ? var.grq_aws_es_host : aws_instance.grq.private_ip}:${var.grq_aws_es ? var.grq_aws_es_port : 9200}"
 
@@ -73,9 +73,6 @@ resource "null_resource" "download_lambdas" {
   provisioner "local-exec" {
     command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data-subscriber-query_handler_package_name}-${var.lambda_package_release}.zip"
   }
-#  provisioner "local-exec" {
-#    command = "curl ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data_subscriber_download_handler_package_name}-${var.lambda_package_release}.zip -o ${var.lambda_data_subscriber_download_handler_package_name}-${var.lambda_package_release}.zip"
-#  }
 }
 
 resource "null_resource" "is_cnm_r_event_trigger_value_valid" {
@@ -1006,7 +1003,6 @@ resource "aws_instance" "mozart" {
       "echo >> ~/.sds/config",
 
       "echo GIT_OAUTH_TOKEN: ${var.git_auth_key} >> ~/.sds/config",
-#	  "echo PUT_GIT_OAUTH_TOKEN: ${var.pub_git_auth_key} >> ~/.sds/config",
       "echo >> ~/.sds/config",
 
       "echo PROVES_URL: https://prov-es.jpl.nasa.gov/beta >> ~/.sds/config",
@@ -1074,7 +1070,6 @@ resource "aws_instance" "mozart" {
       "  ./install.sh mozart -d",
       "  rm -rf ~/mozart/pkgs/hysds-verdi-latest.tar.gz",
       "else",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
@@ -1091,10 +1086,10 @@ resource "aws_instance" "mozart" {
       "fi",
       "cd ~/mozart/ops",
       "if [ \"${var.use_artifactory}\" = true ]; then",
-      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-pcm-${var.pcm_branch}.tar.gz\"",
-      "  tar xfz ${var.project}-sds-pcm-${var.pcm_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-pcm-${var.pcm_branch} /export/home/hysdsops/mozart/ops/${var.project}-pcm",
-      "  rm -rf ${var.project}-sds-pcm-${var.pcm_branch}.tar.gz ",
+      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-pcm-${var.pcm_branch}.tar.gz\"",
+      "  tar xfz ${var.project}-pcm-${var.pcm_branch}.tar.gz",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-pcm-${var.pcm_branch} /export/home/hysdsops/mozart/ops/${var.project}-pcm",
+      "  rm -rf ${var.project}-pcm-${var.pcm_branch}.tar.gz ",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/CNM_product_delivery-${var.product_delivery_branch}.tar.gz\"",
       "  tar xfz CNM_product_delivery-${var.product_delivery_branch}.tar.gz",
       "  ln -s /export/home/hysdsops/mozart/ops/CNM_product_delivery-${var.product_delivery_branch} /export/home/hysdsops/mozart/ops/CNM_product_delivery",
@@ -1105,7 +1100,7 @@ resource "aws_instance" "mozart" {
       "  rm -rf pcm_commons-${var.pcm_commons_branch}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/bach-api",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/bach-api",
       "  rm -rf ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz ",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz",
@@ -1121,6 +1116,8 @@ resource "aws_instance" "mozart" {
       "export PATH=~/conda/bin:$PATH",
       "cp -rp ${var.project}-pcm/conf/sds ~/.sds",
       "cp ~/.sds.bak/config ~/.sds",
+      # create the data subscriber catalog elasticsearch index
+      "python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
       "cd bach-ui",
       "~/conda/bin/npm install --silent --no-progress",
       "sh create_config_simlink.sh ~/.sds/config ~/mozart/ops/bach-ui",
@@ -1253,8 +1250,8 @@ resource "aws_instance" "mozart" {
 #      "echo Build container",
 #      "if [ \"${var.use_artifactory}\" = true ]; then",
 #      "    ~/mozart/ops/${var.project}-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/hysds_pkgs/container-nasa_${var.project}-sds-pcm-${var.pcm_branch}.sdspkg.tar",
-#      "    sds pkg import container-nasa_${project}-pcm-${pcm_branch}.sdspkg.tar",
-#      "    rm -rf container-nasa_${project}-pcm-${pcm_branch}.sdspkg.tar",
+#      "    sds pkg import container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
+#      "    rm -rf container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
 #      "    fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:\"container-nasa_${var.project}-sds-pcm:${lower(var.pcm_branch)}\"",
 #      "else",
 #      "    sds -d ci add_job -b ${var.pcm_branch} --token https://${var.pcm_repo} s3",
@@ -1557,14 +1554,12 @@ resource "aws_instance" "metrics" {
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "fi"
@@ -1625,14 +1620,12 @@ resource "aws_instance" "grq" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-grq_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-grq_venv-${var.hysds_release}.tar.gz",
@@ -1715,14 +1708,12 @@ resource "aws_instance" "factotum" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#     "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-verdi_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-verdi_venv-${var.hysds_release}.tar.gz",
@@ -1986,7 +1977,7 @@ resource "aws_lambda_function" "data_subscriber_download_timer" {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
       "JOB_QUEUE": "opera-job_worker-small",
-      "JOB_TYPE": local.data_subscriber_download_job_type,
+      "JOB_TYPE": local.data_download_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
       "ISL_STAGING_AREA": var.isl_staging_area,
@@ -2006,7 +1997,7 @@ resource "aws_cloudwatch_log_group" "data_subscriber_download_timer" {
 resource "aws_cloudwatch_event_rule" "data_subscriber_download_timer" {
   name = "${aws_lambda_function.data_subscriber_download_timer.function_name}-Trigger"
   description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
-  schedule_expression = var.data_subscriber_download_timer_trigger_frequency
+  schedule_expression = var.data_download_timer_trigger_frequency
   is_enabled = local.enable_timer
 }
 
@@ -2041,7 +2032,7 @@ resource "aws_lambda_function" "data_subscriber_query_timer" {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
       "JOB_QUEUE": "factotum-job_worker-small",
-      "JOB_TYPE": local.data_subscriber_query_job_type,
+      "JOB_TYPE": local.data_query_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
       "ISL_STAGING_AREA": var.isl_staging_area,
@@ -2060,7 +2051,7 @@ resource "aws_cloudwatch_log_group" "data_subscriber_query_timer" {
 resource "aws_cloudwatch_event_rule" "data_subscriber_query_timer" {
   name = "${aws_lambda_function.data_subscriber_query_timer.function_name}-Trigger"
   description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
-  schedule_expression = var.data_subscriber_query_timer_trigger_frequency
+  schedule_expression = var.data_query_timer_trigger_frequency
   is_enabled = local.enable_timer
 }
 

From a395f17c21ae18c517f786da1b63d8a998867e52 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Mon, 4 Apr 2022 10:03:34 -0700
Subject: [PATCH 40/48] Add data subscriber frequency to other venue terraform
 script

---
 cluster_provisioning/dev-e2e-baseline-pge/main.tf       | 3 ++-
 cluster_provisioning/dev-e2e-baseline-pge/variables.tf  | 8 ++++++--
 cluster_provisioning/dev-e2e-event-misfire/variables.tf | 8 ++++++--
 cluster_provisioning/dev-e2e-pge/main.tf                | 5 +++--
 cluster_provisioning/dev-e2e-pge/variables.tf           | 8 ++++++--
 cluster_provisioning/dev-e2e/main.tf                    | 8 ++++----
 cluster_provisioning/dev-e2e/variables.tf               | 4 ++--
 cluster_provisioning/dev-int/main.tf                    | 3 ++-
 cluster_provisioning/dev-releaser/main.tf               | 3 ++-
 cluster_provisioning/dev-releaser/variables.tf          | 6 +++++-
 cluster_provisioning/dev-restore-snapshot/main.tf       | 3 ++-
 cluster_provisioning/dev-restore-snapshot/variables.tf  | 6 +++++-
 cluster_provisioning/dev/main.tf                        | 4 ++--
 cluster_provisioning/dev/variables.tf                   | 6 +++---
 cluster_provisioning/int/variables.tf                   | 8 ++++++--
 cluster_provisioning/modules/common/main.tf             | 5 +++--
 cluster_provisioning/modules/common/variables.tf        | 6 +++---
 17 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/cluster_provisioning/dev-e2e-baseline-pge/main.tf b/cluster_provisioning/dev-e2e-baseline-pge/main.tf
index cf1b3f8b5..afaa199b6 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/main.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
index 21215b6a1..9b089f442 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
@@ -369,7 +369,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -429,4 +433,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e-event-misfire/variables.tf b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
index d3be83945..54cf0d55b 100644
--- a/cluster_provisioning/dev-e2e-event-misfire/variables.tf
+++ b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
@@ -361,7 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -421,4 +425,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e-pge/main.tf b/cluster_provisioning/dev-e2e-pge/main.tf
index 640c74a8f..65221d0cd 100644
--- a/cluster_provisioning/dev-e2e-pge/main.tf
+++ b/cluster_provisioning/dev-e2e-pge/main.tf
@@ -75,8 +75,9 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
-  #obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
+  obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
   code_bucket                             = var.code_bucket
diff --git a/cluster_provisioning/dev-e2e-pge/variables.tf b/cluster_provisioning/dev-e2e-pge/variables.tf
index 82fb2e85e..a2fcc0a1d 100644
--- a/cluster_provisioning/dev-e2e-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-pge/variables.tf
@@ -407,7 +407,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -471,4 +475,4 @@ variable "earthdata_pass" {
 
 variable "pge_test_package" {
   default = "testdata_R1.0.0"
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/dev-e2e/main.tf b/cluster_provisioning/dev-e2e/main.tf
index aa027b001..f75f50032 100644
--- a/cluster_provisioning/dev-e2e/main.tf
+++ b/cluster_provisioning/dev-e2e/main.tf
@@ -73,8 +73,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
-  obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
   code_bucket                             = var.code_bucket
@@ -93,8 +93,8 @@ module "common" {
   es_snapshot_bucket                      = var.es_snapshot_bucket
   es_bucket_role_arn                      = var.es_bucket_role_arn
   run_smoke_test                          = var.run_smoke_test
-  data_subscriber_download_timer_trigger_frequency = ""
-  data_subscriber_query_timer_trigger_frequency = ""
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
 }
 
 locals {
diff --git a/cluster_provisioning/dev-e2e/variables.tf b/cluster_provisioning/dev-e2e/variables.tf
index 5ea05610f..48e8dc7fd 100644
--- a/cluster_provisioning/dev-e2e/variables.tf
+++ b/cluster_provisioning/dev-e2e/variables.tf
@@ -361,11 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_download_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
-variable "data_subscriber_query_timer_trigger_frequency" {
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev-int/main.tf b/cluster_provisioning/dev-int/main.tf
index f3409a91b..af52747c0 100644
--- a/cluster_provisioning/dev-int/main.tf
+++ b/cluster_provisioning/dev-int/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-releaser/main.tf b/cluster_provisioning/dev-releaser/main.tf
index f7b2cc640..973784926 100644
--- a/cluster_provisioning/dev-releaser/main.tf
+++ b/cluster_provisioning/dev-releaser/main.tf
@@ -71,7 +71,8 @@ module "common" {
   use_daac_cnm                            = var.use_daac_cnm
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-releaser/variables.tf b/cluster_provisioning/dev-releaser/variables.tf
index 34ab32eec..77670759b 100644
--- a/cluster_provisioning/dev-releaser/variables.tf
+++ b/cluster_provisioning/dev-releaser/variables.tf
@@ -346,7 +346,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev-restore-snapshot/main.tf b/cluster_provisioning/dev-restore-snapshot/main.tf
index 43818bee1..5f2130a0c 100644
--- a/cluster_provisioning/dev-restore-snapshot/main.tf
+++ b/cluster_provisioning/dev-restore-snapshot/main.tf
@@ -74,7 +74,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_timer_trigger_frequency = var.data_subscriber_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev-restore-snapshot/variables.tf b/cluster_provisioning/dev-restore-snapshot/variables.tf
index e3859dac5..1eef21a04 100644
--- a/cluster_provisioning/dev-restore-snapshot/variables.tf
+++ b/cluster_provisioning/dev-restore-snapshot/variables.tf
@@ -361,7 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
diff --git a/cluster_provisioning/dev/main.tf b/cluster_provisioning/dev/main.tf
index aa810f384..96966b9e7 100644
--- a/cluster_provisioning/dev/main.tf
+++ b/cluster_provisioning/dev/main.tf
@@ -73,8 +73,8 @@ module "common" {
   pge_release                             = var.pge_release
   crid                                    = var.crid
   cluster_type                            = var.cluster_type
-  data_subscriber_download_timer_trigger_frequency = var.data_subscriber_download_timer_trigger_frequency
-  data_subscriber_query_timer_trigger_frequency = var.data_subscriber_query_timer_trigger_frequency
+  data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
+  data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
   obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
diff --git a/cluster_provisioning/dev/variables.tf b/cluster_provisioning/dev/variables.tf
index 3c0470507..fcc4ae97f 100644
--- a/cluster_provisioning/dev/variables.tf
+++ b/cluster_provisioning/dev/variables.tf
@@ -361,11 +361,11 @@ variable "cluster_type" {
   default = "reprocessing"
 }
 
-variable "data_subscriber_download_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
-variable "data_subscriber_query_timer_trigger_frequency" {
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -433,4 +433,4 @@ variable "earthdata_user" {
 
 variable "earthdata_pass" {
   default = ""
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/int/variables.tf b/cluster_provisioning/int/variables.tf
index 2f42e62c3..14cd15f6e 100644
--- a/cluster_provisioning/int/variables.tf
+++ b/cluster_provisioning/int/variables.tf
@@ -330,7 +330,11 @@ variable "cluster_type" {
   default = ""
 }
 
-variable "data_subscriber_timer_trigger_frequency" {
+variable "data_download_timer_trigger_frequency" {
+  default = "rate(60 minutes)"
+}
+
+variable "data_query_timer_trigger_frequency" {
   default = "rate(60 minutes)"
 }
 
@@ -455,4 +459,4 @@ variable "queues" {
       "total_jobs_metric" = true
     }
   }
-}
\ No newline at end of file
+}
diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 4595b2af3..122b5b27a 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -1116,8 +1116,6 @@ resource "aws_instance" "mozart" {
       "export PATH=~/conda/bin:$PATH",
       "cp -rp ${var.project}-pcm/conf/sds ~/.sds",
       "cp ~/.sds.bak/config ~/.sds",
-      # create the data subscriber catalog elasticsearch index
-      "python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
       "cd bach-ui",
       "~/conda/bin/npm install --silent --no-progress",
       "sh create_config_simlink.sh ~/.sds/config ~/mozart/ops/bach-ui",
@@ -1170,6 +1168,9 @@ resource "aws_instance" "mozart" {
       "pip install --progress-bar off -e .",
       "cd ~/mozart/ops/opera-pcm",
       "pip install --progress-bar off -e .",
+      # TODO hyunlee: remove comment after test
+      # create the data subscriber catalog elasticsearch index
+      "python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
       "if [[ \"${var.pge_release}\" == \"develop\"* ]]; then",
       "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_dev_url}",
       "else",
diff --git a/cluster_provisioning/modules/common/variables.tf b/cluster_provisioning/modules/common/variables.tf
index 363d0e3e5..054ccf1ed 100644
--- a/cluster_provisioning/modules/common/variables.tf
+++ b/cluster_provisioning/modules/common/variables.tf
@@ -350,11 +350,11 @@ variable "crid" {
   default = "D00100"
 }
 
-variable "obs_acct_report_timer_trigger_frequency" {}
+variable "data_download_timer_trigger_frequency" {}
 
-variable "data_subscriber_download_timer_trigger_frequency" {}
+variable "data_query_timer_trigger_frequency" {}
 
-variable "data_subscriber_query_timer_trigger_frequency" {}
+variable "obs_acct_report_timer_trigger_frequency" {}
 
 variable "cluster_type" {}
 

From 8b48b34ef87fc2283bdd002be5f8a314deb1f6e4 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Mon, 4 Apr 2022 18:16:29 -0700
Subject: [PATCH 41/48] Add bach-api & bach-ui status check in the
 run_smoke_test.sh

---
 cluster_provisioning/modules/common/main.tf   | 32 ++++++++++-------
 .../modules/common/variables.tf               |  4 +--
 cluster_provisioning/run_smoke_test.sh        | 36 ++++++++++---------
 3 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 122b5b27a..cd9132a7e 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -21,8 +21,9 @@ locals {
   daac_delivery_region              = split(":", var.daac_delivery_proxy)[3]
   daac_delivery_account             = split(":", var.daac_delivery_proxy)[4]
   daac_delivery_resource_name       = split(":", var.daac_delivery_proxy)[5]
-  pge_artifactory_dev_url           = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/pge_snapshots/${var.pge_snapshots_date}"
-  pge_artifactory_release_url       = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pge/"
+  pge_artifactory_dev_url           = "${var.artifactory_base_url}/general/gov/nasa/jpl/${var.project}/sds/pge/"
+#  pge_artifactory_dev_url           = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/pge_snapshots/${var.pge_snapshots_date}"
+  pge_artifactory_release_url       = "${var.artifactory_base_url}/general/gov/nasa/jpl/${var.project}/sds/pge/"
   daac_proxy_cnm_r_sns_count        = var.environment == "dev" && var.venue != "int" && local.sqs_count == 1 ? 1 : 0
   maturity                          = split("-", var.daac_delivery_proxy)[5]
   timer_handler_job_type            = "timer_handler"
@@ -1168,22 +1169,27 @@ resource "aws_instance" "mozart" {
       "pip install --progress-bar off -e .",
       "cd ~/mozart/ops/opera-pcm",
       "pip install --progress-bar off -e .",
-      # TODO hyunlee: remove comment after test
-      # create the data subscriber catalog elasticsearch index
-      "python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
+      #"if [[ \"${var.pcm_release}\" == \"develop\"* ]]; then",
+      # TODO hyunlee: remove comment after test, we should only create the data_subscriber_catalog when the catalog exists
+      # create the data subscriber catalog elasticsearch index, delete the existing catalog first
+      #"    python ~/mozart/ops/opera-pcm/data_subscriber/delete_catalog.py"
+      #"    python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
+      #"fi",
+      # deploy PGE for R1 (DSWx_HLS)
       "if [[ \"${var.pge_release}\" == \"develop\"* ]]; then",
-      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_dev_url}",
+      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" \\",
+      "    --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_dev_url} \\",
+      "    --username ${var.artifactory_fn_user} --api_key ${var.artifactory_fn_api_key}",
       "else",
-      # TODO chrisjrd: remove
-#      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_release_url}",
       # TODO chrisjrd: extract vars as needed
       "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py \\",
-      "    --image_names opera_pge-dswx_hls \\",
-      "    --pge_release \"1.0.0-er.2.0\" \\",
+      "    --image_names ${var.pge_names} \\",
+#      "    --image_names opera_pge-dswx_hls \\",
+      "    --pge_release \"${var.pge_release}\" \\",
       "    --sds_config ~/.sds/config \\",
-      "    --processes 4 \\",
-      "    --force \\",
-      "    --artifactory_url https://artifactory-fn.jpl.nasa.gov/artifactory/general/gov/nasa/jpl/opera/sds/pge \\",
+      "    --processes 4 --force \\",
+      "    --artifactory_url ${local.pge_artifactory_release_url} \\",
+#      "    --artifactory_url https://artifactory-fn.jpl.nasa.gov/artifactory/general/gov/nasa/jpl/opera/sds/pge \\",
       "    --username ${var.artifactory_fn_user} \\",
       "    --api_key ${var.artifactory_fn_api_key}",
       "fi",
diff --git a/cluster_provisioning/modules/common/variables.tf b/cluster_provisioning/modules/common/variables.tf
index 054ccf1ed..e99ac5c38 100644
--- a/cluster_provisioning/modules/common/variables.tf
+++ b/cluster_provisioning/modules/common/variables.tf
@@ -331,7 +331,7 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_names" {
-  default = "opera_pge-l3_dswx_hls"
+  default = "opera_pge-dswx_hls"
 }
 
 variable "docker_registry_bucket" {
@@ -343,7 +343,7 @@ variable "pge_snapshots_date" {
 }
 
 variable "pge_release" {
-  default = "R1.0.0"
+  default = "1.0.0-er.2.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/run_smoke_test.sh b/cluster_provisioning/run_smoke_test.sh
index be6f2210e..a1e724d82 100755
--- a/cluster_provisioning/run_smoke_test.sh
+++ b/cluster_provisioning/run_smoke_test.sh
@@ -22,6 +22,8 @@ if [ "$#" -eq 19 ]; then
   use_daac_cnm=${17}
   crid=${18}
   cluster_type=${19}
+#  data_query_timer_trigger_frequency=${20}
+#  data_download_timer_trigger_frequency=${21}
 else
   echo "Invalid number or arguments ($#) $*" 1>&2
   exit 1
@@ -164,19 +166,19 @@ data_end="${tomorrow}T00:00:00"
 #python ~/mozart/ops/opera-pcm/report/accountability_report_cli.py ObservationAccountabilityReport --start ${start_date_time} --end ${end_date_time} --format_type=xml
 #cat oad_*.xml
 
-#opera_bach_ui_status_code=$(curl -k --write-out %{http_code} --silent --output /dev/null https://${mozart_private_ip}/bach_ui/2.0/data-summary/incoming)
-#opera_bach_api_status_code=$(curl -k --write-out %{http_code} --silent --output /dev/null https://${mozart_private_ip}/bach-api/2.0/ancillary/list)
+opera_bach_ui_status_code=$(curl -k --write-out %{http_code} --silent --output /dev/null https://${mozart_private_ip}/bach-ui/data-summary/incoming)
+opera_bach_api_status_code=$(curl -k --write-out %{http_code} --silent --output /dev/null https://${mozart_private_ip}/bach-api/ancillary/list)
 
-#if [[ "$opera_bach_ui_status_code" -ne 200 ]] ; then
-#  echo "FAILURE: Could not reach bach_ui v2.0" > /tmp/opera_bach_ui_status_code.txt
-#else
-#  echo "SUCCESS" > /tmp/opera_bach_ui_status_code.txt
-#fi
-#if [[ "$opera_bach_api_status_code" -ne 200 ]] ; then
-#  echo "FAILURE: Could not reach bach-api v2.0" > /tmp/opera_bach_api_status_code.txt
-#else
-#  echo "SUCCESS" > /tmp/opera_bach_api_status_code.txt
-#fi
+if [[ "$opera_bach_ui_status_code" -ne 200 ]] ; then
+  echo "FAILURE: Could not reach bach-ui" > /tmp/opera_bach_ui_status_code.txt
+else
+  echo "SUCCESS" > /tmp/opera_bach_ui_status_code.txt
+fi
+if [[ "$opera_bach_api_status_code" -ne 200 ]] ; then
+  echo "FAILURE: Could not reach bach-api" > /tmp/opera_bach_api_status_code.txt
+else
+  echo "SUCCESS" > /tmp/opera_bach_api_status_code.txt
+fi
 
 # Test auto generation of the Observation Accountability Report
 #if [ "${cluster_type}" = "forward" ]; then
@@ -188,17 +190,19 @@ data_end="${tomorrow}T00:00:00"
 #fi
 
 # If we're deploying a forward cluster, restore the original settings.yaml to the cluster
-#if [ "${cluster_type}" = "forward" ]; then
-#  aws events put-rule --name ${project}-${venue}-${counter}-l0a-timer-Trigger --schedule-expression "${l0a_timer_trigger_frequency}"
+if [ "${cluster_type}" = "forward" ]; then
+  aws events put-rule --name ${project}-${venue}-${counter}-data-subscriber-query-timer-Trigger --schedule-expression "rate(5 minutes)"
+  aws events put-rule --name ${project}-${venue}-${counter}-data-subscriber-download-timer-Trigger --schedule-expression "rate(5 minutes)"
+#  aws events put-rule --name ${project}-${venue}-${counter}-data-subscriber-query-timer-Trigger --schedule-expression "${data_query_timer_trigger_frequency}"
+#  aws events put-rule --name ${project}-${venue}-${counter}-data-subscriber-download-timer-Trigger --schedule-expression "${data_download_timer_trigger_frequency}"
 #  python ~/mozart/ops/opera-pcm/conf/sds/files/test/check_forced_state_configs.py datasets_e2e_force_submits.json LDF /tmp/check_expected_force_submits.txt
-
 #  echo "Restoring original settings.yaml and pushing it out to the cluster"
 #  cp ~/mozart/ops/opera-pcm/conf/settings.yaml.bak ~/mozart/ops/opera-pcm/conf/settings.yaml
 #  fab -f ~/.sds/cluster.py -R mozart,grq,factotum update_opera_packages
 #  sds ship
 #else
 #  echo "SUCCESS: No force submit state configs expected to be found." > /tmp/check_expected_force_submits.txt
-#fi
+fi
 
 # If we're deploying a forward cluster, restore the original settings that will create the daily observational
 # accountability reports

From 3a87b4928b32eed03cc7d266ac00b13c457be2b2 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Mon, 4 Apr 2022 18:34:58 -0700
Subject: [PATCH 42/48] Add log type as l3_dswx_hls outputs

---
 cluster_provisioning/ebs-snapshot/variables.tf | 4 ++--
 conf/pge_outputs.yaml                          | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cluster_provisioning/ebs-snapshot/variables.tf b/cluster_provisioning/ebs-snapshot/variables.tf
index 776242a1d..93b61be7f 100644
--- a/cluster_provisioning/ebs-snapshot/variables.tf
+++ b/cluster_provisioning/ebs-snapshot/variables.tf
@@ -13,7 +13,7 @@ variable "profile" {
 }
 
 variable "verdi_release" {
-  #default = "v4.0.1-beta.8"
+  #default = "v4.0.1-beta.8-oraclelinux"
 }
 
 variable "registry_release" {
@@ -29,7 +29,7 @@ variable "pge_snapshots_date" {
 }
 
 variable "pge_release" {
-  default = "v1.0.0-er.2.0"
+  default = "v1.0.0-er.3.0"
 }
 
 variable "private_key_file" {
diff --git a/conf/pge_outputs.yaml b/conf/pge_outputs.yaml
index 6b4000f6f..839c6a931 100644
--- a/conf/pge_outputs.yaml
+++ b/conf/pge_outputs.yaml
@@ -41,6 +41,7 @@ L3_HLS:
       - regex: !!python/regexp '(?P<id>(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)_(?P<source>HLS)_(?P<sensor>SENTINEL-2A|LANDSAT-8)_(?P<tile_id>T[^\W_]{5})_(?P<datetime>(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})T(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2}))_(?P<collection_version>v\d+[.]\d+)[.](?P<ext>tif))$'
         verify: true
         hash: md5
-    Secondary: []
+    Secondary:
+      - regex: !!python/regexp '(?P<id>(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)_(?P<source>HLS)_(?P<sensor>SENTINEL-2A|LANDSAT-8)_(?P<tile_id>T[^\W_]{5})_(?P<datetime>(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})T(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2}))_(?P<collection_version>v\d+[.]\d+)[.](?P<ext>log))$'
     Optional: []
   Missing_Metadata: {}

From 36fd4ef6934acb9bc6a47248b4a7324c27d5f9b2 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Tue, 5 Apr 2022 12:39:25 -0700
Subject: [PATCH 43/48] Fix for merge conflict with develop branch

---
 cluster_provisioning/dev-int/versions.tf    |  22 +-
 cluster_provisioning/modules/common/main.tf | 257 ++++++++++++--------
 2 files changed, 158 insertions(+), 121 deletions(-)
 mode change 100644 => 120000 cluster_provisioning/dev-int/versions.tf

diff --git a/cluster_provisioning/dev-int/versions.tf b/cluster_provisioning/dev-int/versions.tf
deleted file mode 100644
index e465dbaa8..000000000
--- a/cluster_provisioning/dev-int/versions.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-terraform {
-  required_version = ">= 1.0.11"
-
-  required_providers {
-    # TODO: remove this pin once this ticket is resolved:
-    #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    random = {
-      source = "hashicorp/random"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev-int/versions.tf b/cluster_provisioning/dev-int/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev-int/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 6254b5be7..54ea628eb 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -1,35 +1,37 @@
 locals {
-  counter                        = var.counter != "" ? var.counter : random_id.counter.hex
-  default_dataset_bucket         = "${var.project}-${var.environment}-rs-fwd-${var.venue}"
-  dataset_bucket                 = var.dataset_bucket != "" ? var.dataset_bucket : local.default_dataset_bucket
-  default_code_bucket            = "${var.project}-${var.environment}-cc-fwd-${var.venue}"
-  code_bucket                    = var.code_bucket != "" ? var.code_bucket : local.default_code_bucket
-  default_isl_bucket             = "${var.project}-${var.environment}-isl-fwd-${var.venue}"
-  isl_bucket                     = var.isl_bucket != "" ? var.isl_bucket : local.default_isl_bucket
-  default_osl_bucket             = "${var.project}-${var.environment}-osl-fwd-${var.venue}"
-  osl_bucket                     = var.osl_bucket != "" ? var.osl_bucket : local.default_osl_bucket
-  default_triage_bucket          = "${var.project}-${var.environment}-triage-fwd-${var.venue}"
-  triage_bucket                  = var.triage_bucket != "" ? var.triage_bucket : local.default_triage_bucket
-  default_lts_bucket             = "${var.project}-${var.environment}-lts-fwd-${var.venue}"
-  lts_bucket                     = var.lts_bucket != "" ? var.lts_bucket : local.default_lts_bucket
-  key_name                       = var.keypair_name != "" ? var.keypair_name : split(".", basename(var.private_key_file))[0]
-  sns_count                      = var.cnm_r_event_trigger == "sns" ? 1 : 0
-  kinesis_count                  = var.cnm_r_event_trigger == "kinesis" ? 1 : 0
-  sqs_count                      = var.cnm_r_event_trigger == "sqs" ? 1 : 0
-  lambda_repo                    = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/lambda"
-  daac_delivery_event_type       = split(":", var.daac_delivery_proxy)[2]
-  daac_delivery_region           = split(":", var.daac_delivery_proxy)[3]
-  daac_delivery_account          = split(":", var.daac_delivery_proxy)[4]
-  daac_delivery_resource_name    = split(":", var.daac_delivery_proxy)[5]
-  pge_artifactory_dev_url        = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/pge_snapshots/${var.pge_snapshots_date}"
-  pge_artifactory_release_url    = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pge/"
-  daac_proxy_cnm_r_sns_count     = var.environment == "dev" && var.venue != "int" && local.sqs_count == 1 ? 1 : 0
-  maturity                       = split("-", var.daac_delivery_proxy)[5]
-  timer_handler_job_type         = "timer_handler"
-  accountability_report_job_type = "accountability_report"
-  data_subscriber_job_type       = "data_subscriber"
-  use_s3_uri_structure           = var.use_s3_uri_structure
-  grq_es_url                     = "${var.grq_aws_es ? "https" : "http"}://${var.grq_aws_es ? var.grq_aws_es_host : aws_instance.grq.private_ip}:${var.grq_aws_es ? var.grq_aws_es_port : 9200}"
+  counter                           = var.counter != "" ? var.counter : random_id.counter.hex
+  default_dataset_bucket            = "${var.project}-${var.environment}-rs-fwd-${var.venue}"
+  dataset_bucket                    = var.dataset_bucket != "" ? var.dataset_bucket : local.default_dataset_bucket
+  default_code_bucket               = "${var.project}-${var.environment}-cc-fwd-${var.venue}"
+  code_bucket                       = var.code_bucket != "" ? var.code_bucket : local.default_code_bucket
+  default_isl_bucket                = "${var.project}-${var.environment}-isl-fwd-${var.venue}"
+  isl_bucket                        = var.isl_bucket != "" ? var.isl_bucket : local.default_isl_bucket
+  default_osl_bucket                = "${var.project}-${var.environment}-osl-fwd-${var.venue}"
+  osl_bucket                        = var.osl_bucket != "" ? var.osl_bucket : local.default_osl_bucket
+  default_triage_bucket             = "${var.project}-${var.environment}-triage-fwd-${var.venue}"
+  triage_bucket                     = var.triage_bucket != "" ? var.triage_bucket : local.default_triage_bucket
+  default_lts_bucket                = "${var.project}-${var.environment}-lts-fwd-${var.venue}"
+  lts_bucket                        = var.lts_bucket != "" ? var.lts_bucket : local.default_lts_bucket
+  key_name                          = var.keypair_name != "" ? var.keypair_name : split(".", basename(var.private_key_file))[0]
+  sns_count                         = var.cnm_r_event_trigger == "sns" ? 1 : 0
+  kinesis_count                     = var.cnm_r_event_trigger == "kinesis" ? 1 : 0
+  sqs_count                         = var.cnm_r_event_trigger == "sqs" ? 1 : 0
+  lambda_repo                       = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/lambda"
+  daac_delivery_event_type          = split(":", var.daac_delivery_proxy)[2]
+  daac_delivery_region              = split(":", var.daac_delivery_proxy)[3]
+  daac_delivery_account             = split(":", var.daac_delivery_proxy)[4]
+  daac_delivery_resource_name       = split(":", var.daac_delivery_proxy)[5]
+  pge_artifactory_dev_url           = "${var.artifactory_base_url}/general/gov/nasa/jpl/${var.project}/sds/pge/"
+#  pge_artifactory_dev_url           = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/pge_snapshots/${var.pge_snapshots_date}"
+  pge_artifactory_release_url       = "${var.artifactory_base_url}/general/gov/nasa/jpl/${var.project}/sds/pge/"
+  daac_proxy_cnm_r_sns_count        = var.environment == "dev" && var.venue != "int" && local.sqs_count == 1 ? 1 : 0
+  maturity                          = split("-", var.daac_delivery_proxy)[5]
+  timer_handler_job_type            = "timer_handler"
+  accountability_report_job_type    = "accountability_report"
+  data_download_job_type            = "data_subscriber_download"
+  data_query_job_type               = "data_subscriber_query"
+  use_s3_uri_structure              = var.use_s3_uri_structure
+  grq_es_url                        = "${var.grq_aws_es ? "https" : "http"}://${var.grq_aws_es ? var.grq_aws_es_host : aws_instance.grq.private_ip}:${var.grq_aws_es ? var.grq_aws_es_port : 9200}"
 
 
   cnm_response_queue_name = {
@@ -67,11 +69,11 @@ resource "null_resource" "download_lambdas" {
     command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_report_handler_package_name}-${var.lambda_package_release}.zip"
   }
   provisioner "local-exec" {
-    command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data-subscriber_handler_package_name}-${var.lambda_package_release}.zip"
+    command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data-subscriber-download_handler_package_name}-${var.lambda_package_release}.zip"
+  }
+  provisioner "local-exec" {
+    command = "curl -H \"X-JFrog-Art-Api:${var.artifactory_fn_api_key}\" -O ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data-subscriber-query_handler_package_name}-${var.lambda_package_release}.zip"
   }
-#  provisioner "local-exec" {
-#    command = "curl ${local.lambda_repo}/${var.lambda_package_release}/${var.lambda_data_subscriber_handler_package_name}-${var.lambda_package_release}.zip -o ${var.lambda_data_subscriber_handler_package_name}-${var.lambda_package_release}.zip"
-#  }
 }
 
 resource "null_resource" "is_cnm_r_event_trigger_value_valid" {
@@ -735,7 +737,7 @@ resource "aws_lambda_event_source_mapping" "isl_queue_event_source_mapping" {
 }
 
 #####################################
-# sds config  QUEUE block generation 
+# sds config  QUEUE block generation
 #####################################
 data "template_file" "config" {
   template = file("${path.module}/config.tmpl")
@@ -1002,7 +1004,6 @@ resource "aws_instance" "mozart" {
       "echo >> ~/.sds/config",
 
       "echo GIT_OAUTH_TOKEN: ${var.git_auth_key} >> ~/.sds/config",
-#	  "echo PUT_GIT_OAUTH_TOKEN: ${var.pub_git_auth_key} >> ~/.sds/config",
       "echo >> ~/.sds/config",
 
       "echo PROVES_URL: https://prov-es.jpl.nasa.gov/beta >> ~/.sds/config",
@@ -1070,19 +1071,16 @@ resource "aws_instance" "mozart" {
       "  ./install.sh mozart -d",
       "  rm -rf ~/mozart/pkgs/hysds-verdi-latest.tar.gz",
       "else",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-mozart_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-mozart_venv-${var.hysds_release}.tar.gz\"",
 
       "  tar xfz hysds-mozart_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-mozart_venv-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-verdi_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-verdi_venv-${var.hysds_release}.tar.gz",
@@ -1103,11 +1101,11 @@ resource "aws_instance" "mozart" {
       "  rm -rf pcm_commons-${var.pcm_commons_branch}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-api",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-bach-api-${var.bach_api_branch} /export/home/hysdsops/mozart/ops/bach-api",
       "  rm -rf ${var.project}-sds-bach-api-${var.bach_api_branch}.tar.gz ",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz\"",
       "  tar xfz ${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz",
-      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-ui-${var.bach_ui_branch} /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-ui",
+      "  ln -s /export/home/hysdsops/mozart/ops/${var.project}-sds-bach-ui-${var.bach_ui_branch} /export/home/hysdsops/mozart/ops/bach-ui",
       "  rm -rf ${var.project}-sds-bach-ui-${var.bach_ui_branch}.tar.gz ",
       "else",
       "  git clone --single-branch -b ${var.pcm_branch} https://${var.git_auth_key}@${var.pcm_repo} ${var.project}-pcm",
@@ -1183,7 +1181,7 @@ resource "aws_instance" "mozart" {
       "if [ \"${var.use_artifactory}\" = true ]; then",
       "  fab -f ~/.sds/cluster.py -R mozart,grq,metrics,factotum update_${var.project}_packages",
       "else",
-      "  fab -f ~/.sds/cluster.py -R mozart,grq,metrics,factotum,verdi update_${var.project}_packages",
+      "  fab -f ~/.sds/cluster.py -R mozart,grq,metrics,factotum update_${var.project}_packages",
       "fi",
       "if [ \"${var.grq_aws_es}\" = true ] && [ \"${var.use_grq_aws_es_private_verdi}\" = true ]; then",
       "  fab -f ~/.sds/cluster.py -R mozart update_celery_config",
@@ -1200,19 +1198,27 @@ resource "aws_instance" "mozart" {
       "pip install --progress-bar off -e .",
       "cd ~/mozart/ops/opera-pcm",
       "pip install --progress-bar off -e .",
+      #"if [[ \"${var.pcm_release}\" == \"develop\"* ]]; then",
+      # TODO hyunlee: remove comment after test, we should only create the data_subscriber_catalog when the catalog exists
+      # create the data subscriber catalog elasticsearch index, delete the existing catalog first
+      #"    python ~/mozart/ops/opera-pcm/data_subscriber/delete_catalog.py"
+      #"    python ~/mozart/ops/opera-pcm/data_subscriber/create_catalog.py",
+      #"fi",
+      # deploy PGE for R1 (DSWx_HLS)
       "if [[ \"${var.pge_release}\" == \"develop\"* ]]; then",
-      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_dev_url}",
+      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" \\",
+      "    --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_dev_url} \\",
+      "    --username ${var.artifactory_fn_user} --api_key ${var.artifactory_fn_api_key}",
       "else",
-      # TODO chrisjrd: remove
-#      "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py --pge_release \"${var.pge_release}\" --image_names ${var.pge_names} --sds_config ~/.sds/config --processes 4 --force --artifactory_url ${local.pge_artifactory_release_url}",
       # TODO chrisjrd: extract vars as needed
       "    python ~/mozart/ops/opera-pcm/tools/deploy_pges.py \\",
-      "    --image_names opera_pge-dswx_hls \\",
-      "    --pge_release \"1.0.0-er.2.0\" \\",
+      "    --image_names ${var.pge_names} \\",
+#      "    --image_names opera_pge-dswx_hls \\",
+      "    --pge_release \"${var.pge_release}\" \\",
       "    --sds_config ~/.sds/config \\",
-      "    --processes 4 \\",
-      "    --force \\",
-      "    --artifactory_url https://artifactory-fn.jpl.nasa.gov/artifactory/general/gov/nasa/jpl/opera/sds/pge \\",
+      "    --processes 4 --force \\",
+      "    --artifactory_url ${local.pge_artifactory_release_url} \\",
+#      "    --artifactory_url https://artifactory-fn.jpl.nasa.gov/artifactory/general/gov/nasa/jpl/opera/sds/pge \\",
       "    --username ${var.artifactory_fn_user} \\",
       "    --api_key ${var.artifactory_fn_api_key}",
       "fi",
@@ -1225,15 +1231,19 @@ resource "aws_instance" "mozart" {
     ]
   }
 
- # provisioner "remote-exec" {
- #   inline = [
- #     "set -ex",
- #     "source ~/.bash_profile",
- #     "wget ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/nisar/sds/pge/testdata_R1.0.0/l0b_small_001.tgz!/input/id_06-00-0101_chirp-parameter_v44.12.xml -O /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/pge/l0b/id_06-00-0101_chirp-parameter_v44.12.xml",
- #     "wget ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/nisar/sds/pge/testdata_R1.0.0/l0b_small_001.tgz!/input/id_01-00-0101_radar-configuration_v44.12.xml -O /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/pge/l0b/id_01-00-0101_radar-configuration_v44.12.xml",
+  # Get test data from the artifactory and put into tests directory
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      "source ~/.bash_profile",
+      "mkdir -p /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/L3_DSWx_HLS_PGE/test-files/",
+      "wget ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/testdata_R1.0.0/hls_l2.tar.gz \\",
+      "     -O /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/L3_DSWx_HLS_PGE/test-files/hls_l2.tar.gz",
+      "cd /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/L3_DSWx_HLS_PGE/test-files/",
+      "tar xfz hls_l2.tar.gz"
  #     "wget ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/nisar/sds/pge/testdata_R1.0.0/l0b_small_001.tgz!/input/id_ff-00-ff01_waveform.xml -O /export/home/hysdsops/mozart/ops/${var.project}-pcm/tests/pge/l0b/id_ff-00-ff01_waveform.xml",
- #   ]
- # }
+    ]
+  }
 
   // creating the snapshot repositories and lifecycles for GRQ mozart and metrics ES
   provisioner "remote-exec" {
@@ -1275,27 +1285,27 @@ resource "null_resource" "install_pcm_and_pges" {
       # build/import opera-pcm
       "echo Build container",
       "if [ \"${var.use_artifactory}\" = true ]; then",
-      "    ~/mozart/ops/opera-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/opera/sds/pcm/hysds_pkgs/container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
-      "    sds pkg import container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
-      "    rm -rf container-iems-sds_opera-pcm-${var.pcm_branch}.sdspkg.tar",
-      "    fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:\"container-iems-sds_opera-pcm:${lower(var.pcm_branch)}\"",
+      "    ~/mozart/ops/${var.project}-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/hysds_pkgs/container-nasa_${var.project}-sds-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    sds pkg import container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    rm -rf container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
+      "    fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:\"container-nasa_${var.project}-sds-pcm:${lower(var.pcm_branch)}\"",
       "else",
-      "    sds --debug ci add_job --branch ${var.pcm_branch} --token https://${var.pcm_repo} s3",
-      "    sds --debug ci build_job --branch ${var.pcm_branch} https://${var.pcm_repo}",
-      "    sds --debug ci remove_job --branch ${var.pcm_branch} https://${var.pcm_repo}",
+      "    sds -d ci add_job -b ${var.pcm_branch} --token https://${var.pcm_repo} s3",
+      "    sds -d ci build_job -b ${var.pcm_branch} https://${var.pcm_repo}",
+      "    sds -d ci remove_job -b ${var.pcm_branch} https://${var.pcm_repo}",
       "fi",
       # build/import CNM product delivery
       "if [ \"${var.use_artifactory}\" = true ]; then",
-      "    ~/mozart/ops/opera-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/opera/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
-      "    sds pkg import container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
-      "    rm -rf container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
+      "    ~/mozart/ops/${var.project}-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${var.product_delivery_branch}.sdspkg.tar",
+      "    sds pkg import container-iems-sds_${var.project}-pcm-${var.product_delivery_branch}.sdspkg.tar",
+      "    rm -rf container-iems-sds_${var.project}-pcm-${var.product_delivery_branch}.sdspkg.tar",
       "else",
-      "    sds --debug ci add_job --branch ${var.product_delivery_branch} --token https://${var.product_delivery_repo} s3",
-      "    sds --debug ci build_job --branch ${var.product_delivery_branch} https://${var.product_delivery_repo}",
-      "    sds --debug ci remove_job --branch ${var.product_delivery_branch} https://${var.product_delivery_repo}",
+      "    sds -d ci add_job -b ${var.product_delivery_branch} --token https://${var.product_delivery_repo} s3",
+      "    sds -d ci build_job -b ${var.product_delivery_branch} https://${var.product_delivery_repo}",
+      "    sds -d ci remove_job -b ${var.product_delivery_branch} https://${var.product_delivery_repo}",
       "fi",
       "echo Set up trigger rules",
-      "sh ~/mozart/ops/opera-pcm/cluster_provisioning/setup_trigger_rules.sh"
+      "sh ~/mozart/ops/${var.project}-pcm/cluster_provisioning/setup_trigger_rules.sh ${aws_instance.mozart.private_ip}"
     ]
   }
 }
@@ -1376,7 +1386,7 @@ resource "null_resource" "rs_fwd_add_lifecycle_rule" {
 }
 
 ############################
-# Autoscaling Group related 
+# Autoscaling Group related
 ############################
 
 data "aws_subnet_ids" "asg_vpc" {
@@ -1578,14 +1588,12 @@ resource "aws_instance" "metrics" {
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-metrics_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-metrics_venv-${var.hysds_release}.tar.gz",
       "fi"
@@ -1645,14 +1653,12 @@ resource "aws_instance" "grq" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-grq_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-grq_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-grq_venv-${var.hysds_release}.tar.gz",
@@ -1734,14 +1740,12 @@ resource "aws_instance" "factotum" {
     inline = [
       "chmod 755 ~/download_artifact.sh",
       "if [ \"${var.hysds_release}\" != \"develop\" ]; then",
-#     "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-conda_env-${var.hysds_release}.tar.gz\"",
       "  mkdir -p ~/conda",
       "  tar xfz hysds-conda_env-${var.hysds_release}.tar.gz -C conda",
       "  export PATH=$HOME/conda/bin:$PATH",
       "  conda-unpack",
       "  rm -rf hysds-conda_env-${var.hysds_release}.tar.gz",
-#      "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/iems/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  ~/download_artifact.sh -m \"${var.artifactory_mirror_url}\" -b \"${var.artifactory_base_url}\" -k \"${var.artifactory_fn_api_key}\" \"${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/${var.hysds_release}/hysds-verdi_venv-${var.hysds_release}.tar.gz\"",
       "  tar xfz hysds-verdi_venv-${var.hysds_release}.tar.gz",
       "  rm -rf hysds-verdi_venv-${var.hysds_release}.tar.gz",
@@ -1987,13 +1991,12 @@ resource "aws_lambda_permission" "event-misfire_lambda" {
 #  function_name = aws_lambda_function.observation_accountability_report_timer.function_name
 #}
 
-# Resources to provision the Data Subscriber timer
-# Lambda function to submit a job to create the Data Subscriber
-resource "aws_lambda_function" "data_subscriber_timer" {
+# Resources to provision the Data Subscriber timers
+resource "aws_lambda_function" "data_subscriber_download_timer" {
   depends_on = [null_resource.download_lambdas]
-  filename = "${var.lambda_data-subscriber_handler_package_name}-${var.lambda_package_release}.zip"
+  filename = "${var.lambda_data-subscriber-download_handler_package_name}-${var.lambda_package_release}.zip"
   description = "Lambda function to submit a job that will create a Data Subscriber"
-  function_name = "${var.project}-${var.venue}-${local.counter}-data-subscriber-timer"
+  function_name = "${var.project}-${var.venue}-${local.counter}-data-subscriber-download-timer"
   handler = "lambda_function.lambda_handler"
   role = var.lambda_role_arn
   runtime = "python3.7"
@@ -2005,8 +2008,8 @@ resource "aws_lambda_function" "data_subscriber_timer" {
   environment {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
-      "JOB_QUEUE": "factotum-job_worker-small",
-      "JOB_TYPE": local.data_subscriber_job_type,
+      "JOB_QUEUE": "opera-job_worker-small",
+      "JOB_TYPE": local.data_download_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
       "ISL_STAGING_AREA": var.isl_staging_area,
@@ -2016,30 +2019,84 @@ resource "aws_lambda_function" "data_subscriber_timer" {
   }
 }
 
-resource "aws_cloudwatch_log_group" "data_subscriber_timer" {
-  depends_on = [aws_lambda_function.data_subscriber_timer]
-  name = "/aws/lambda/${aws_lambda_function.data_subscriber_timer.function_name}"
+resource "aws_cloudwatch_log_group" "data_subscriber_download_timer" {
+  depends_on = [aws_lambda_function.data_subscriber_download_timer]
+  name = "/aws/lambda/${aws_lambda_function.data_subscriber_download_timer.function_name}"
+  retention_in_days = var.lambda_log_retention_in_days
+}
+
+# Cloudwatch event that will trigger a Lambda that submits the Data Subscriber timer job
+resource "aws_cloudwatch_event_rule" "data_subscriber_download_timer" {
+  name = "${aws_lambda_function.data_subscriber_download_timer.function_name}-Trigger"
+  description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
+  schedule_expression = var.data_download_timer_trigger_frequency
+  is_enabled = local.enable_timer
+}
+
+resource "aws_cloudwatch_event_target" "data_subscriber_download_timer" {
+  rule = aws_cloudwatch_event_rule.data_subscriber_download_timer.name
+  target_id = "Lambda"
+  arn = aws_lambda_function.data_subscriber_download_timer.arn
+}
+
+resource "aws_lambda_permission" "data_subscriber_download_timer" {
+  statement_id = aws_cloudwatch_event_rule.data_subscriber_download_timer.name
+  action = "lambda:InvokeFunction"
+  principal = "events.amazonaws.com"
+  source_arn = aws_cloudwatch_event_rule.data_subscriber_download_timer.arn
+  function_name = aws_lambda_function.data_subscriber_download_timer.function_name
+}
+
+resource "aws_lambda_function" "data_subscriber_query_timer" {
+  depends_on = [null_resource.download_lambdas]
+  filename = "${var.lambda_data-subscriber-query_handler_package_name}-${var.lambda_package_release}.zip"
+  description = "Lambda function to submit a job that will create a Data Subscriber"
+  function_name = "${var.project}-${var.venue}-${local.counter}-data-subscriber-query-timer"
+  handler = "lambda_function.lambda_handler"
+  role = var.lambda_role_arn
+  runtime = "python3.7"
+  vpc_config {
+    security_group_ids = [var.cluster_security_group_id]
+    subnet_ids = data.aws_subnet_ids.lambda_vpc.ids
+  }
+  timeout = 30
+  environment {
+    variables = {
+      "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
+      "JOB_QUEUE": "factotum-job_worker-small",
+      "JOB_TYPE": local.data_query_job_type,
+      "JOB_RELEASE": var.pcm_branch,
+      "ISL_BUCKET_NAME": local.isl_bucket,
+      "ISL_STAGING_AREA": var.isl_staging_area,
+      "USER_START_TIME": "",
+      "USER_END_TIME": ""
+    }
+  }
+}
+resource "aws_cloudwatch_log_group" "data_subscriber_query_timer" {
+  depends_on = [aws_lambda_function.data_subscriber_query_timer]
+  name = "/aws/lambda/${aws_lambda_function.data_subscriber_query_timer.function_name}"
   retention_in_days = var.lambda_log_retention_in_days
 }
 
 # Cloudwatch event that will trigger a Lambda that submits the Data Subscriber timer job
-resource "aws_cloudwatch_event_rule" "data_subscriber_timer" {
-  name = "${aws_lambda_function.data_subscriber_timer.function_name}-Trigger"
+resource "aws_cloudwatch_event_rule" "data_subscriber_query_timer" {
+  name = "${aws_lambda_function.data_subscriber_query_timer.function_name}-Trigger"
   description = "Cloudwatch event to trigger the Data Subscriber Timer Lambda"
-  schedule_expression = var.data_subscriber_timer_trigger_frequency
+  schedule_expression = var.data_query_timer_trigger_frequency
   is_enabled = local.enable_timer
 }
 
-resource "aws_cloudwatch_event_target" "data_subscriber_timer" {
-  rule = aws_cloudwatch_event_rule.data_subscriber_timer.name
+resource "aws_cloudwatch_event_target" "data_subscriber_query_timer" {
+  rule = aws_cloudwatch_event_rule.data_subscriber_query_timer.name
   target_id = "Lambda"
-  arn = aws_lambda_function.data_subscriber_timer.arn
+  arn = aws_lambda_function.data_subscriber_query_timer.arn
 }
 
-resource "aws_lambda_permission" "data_subscriber_timer" {
-  statement_id = aws_cloudwatch_event_rule.data_subscriber_timer.name
+resource "aws_lambda_permission" "data_subscriber_query_timer" {
+  statement_id = aws_cloudwatch_event_rule.data_subscriber_query_timer.name
   action = "lambda:InvokeFunction"
   principal = "events.amazonaws.com"
-  source_arn = aws_cloudwatch_event_rule.data_subscriber_timer.arn
-  function_name = aws_lambda_function.data_subscriber_timer.function_name
+  source_arn = aws_cloudwatch_event_rule.data_subscriber_query_timer.arn
+  function_name = aws_lambda_function.data_subscriber_query_timer.function_name
 }

From 65962373a8cedea8976830e44fb090cee7f33797 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Tue, 5 Apr 2022 12:56:14 -0700
Subject: [PATCH 44/48] Comment out pcm and cnm_product_delivery installation
 from run_smoke_test.sh

---
 cluster_provisioning/modules/common/main.tf |  4 +--
 cluster_provisioning/run_smoke_test.sh      | 40 ++++++++++-----------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index d4b23b2e5..0ec602bf3 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -1286,8 +1286,8 @@ resource "null_resource" "install_pcm_and_pges" {
       "echo Build container",
       "if [ \"${var.use_artifactory}\" = true ]; then",
       "    ~/mozart/ops/${var.project}-pcm/tools/download_artifact.sh -m ${var.artifactory_mirror_url} -b ${var.artifactory_base_url} ${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/hysds_pkgs/container-nasa_${var.project}-sds-pcm-${var.pcm_branch}.sdspkg.tar",
-      "    sds pkg import container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
-      "    rm -rf container-nasa_${var.project}-pcm-${var.pcm_branch}.sdspkg.tar",
+	  "    sds pkg import container-nasa_${var.project}-sds-pcm-${var.pcm_branch}.sdspkg.tar", 
+      "    rm -rf container-nasa_${var.project}-sds-pcm-${var.pcm_branch}.sdspkg.tar", 
       "    fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:\"container-nasa_${var.project}-sds-pcm:${lower(var.pcm_branch)}\"",
       "else",
       "    sds -d ci add_job -b ${var.pcm_branch} --token https://${var.pcm_repo} s3",
diff --git a/cluster_provisioning/run_smoke_test.sh b/cluster_provisioning/run_smoke_test.sh
index a1e724d82..dbb2aecb7 100755
--- a/cluster_provisioning/run_smoke_test.sh
+++ b/cluster_provisioning/run_smoke_test.sh
@@ -49,15 +49,15 @@ cnm_datasets=L3_DSWx_HLS
 #fi
 
 # build/import CNM product delivery
-if [ "${use_artifactory}" = true ]; then
-  ~/download_artifact.sh -m ${artifactory_mirror_url} -b ${artifactory_base_url} "${artifactory_base_url}/${artifactory_repo}/gov/nasa/jpl/${project}/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar"
-  sds pkg import container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar
-  rm -rf container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar
-else
-  sds ci add_job -b ${product_delivery_branch} --token https://${product_delivery_repo} s3
-  sds ci build_job -b ${product_delivery_branch} https://${product_delivery_repo}
-  sds ci remove_job -b ${product_delivery_branch} https://${product_delivery_repo}
-fi
+#if [ "${use_artifactory}" = true ]; then
+#  ~/download_artifact.sh -m ${artifactory_mirror_url} -b ${artifactory_base_url} "${artifactory_base_url}/${artifactory_repo}/gov/nasa/jpl/${project}/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar"
+#  sds pkg import container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar
+#  rm -rf container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar
+#else
+#  sds ci add_job -b ${product_delivery_branch} --token https://${product_delivery_repo} s3
+#  sds ci build_job -b ${product_delivery_branch} https://${product_delivery_repo}
+#  sds ci remove_job -b ${product_delivery_branch} https://${product_delivery_repo}
+#fi
 
 cd ~/.sds/files
 
@@ -81,19 +81,19 @@ cd ~/.sds/files
 #aws autoscaling update-auto-scaling-group --auto-scaling-group-name ${project}-${venue}-${counter}-opera-workflow_profiler --desired-capacity 1
 
 # build/import opera-pcm
-lowercase_pcm_branch=`echo "${pcm_branch}" | awk '{ print tolower($0); }'`
+#lowercase_pcm_branch=`echo "${pcm_branch}" | awk '{ print tolower($0); }'`
 
-if [ "${use_artifactory}" = true ]; then
-  ~/download_artifact.sh -m ${artifactory_mirror_url} -b ${artifactory_base_url} "${artifactory_base_url}/${artifactory_repo}/gov/nasa/jpl/${project}/sds/pcm/hysds_pkgs/container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar"
-  sds pkg import container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar
-  rm -rf container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar
+#if [ "${use_artifactory}" = true ]; then
+#  ~/download_artifact.sh -m ${artifactory_mirror_url} -b ${artifactory_base_url} "${artifactory_base_url}/${artifactory_repo}/gov/nasa/jpl/${project}/sds/pcm/hysds_pkgs/container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar"
+#  sds pkg import container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar
+#  rm -rf container-nasa_${project}-sds-pcm-${pcm_branch}.sdspkg.tar
   # Loads the opera-pcm container to the docker registry
-  fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:"container-nasa_${project}-sds-pcm:${lowercase_pcm_branch}"
-else
-  sds -d ci add_job -b ${pcm_branch} --token https://${pcm_repo} s3
-  sds -d ci build_job -b ${pcm_branch} https://${pcm_repo}
-  sds -d ci remove_job -b ${pcm_branch} https://${pcm_repo}
-fi
+#  fab -f ~/.sds/cluster.py -R mozart load_container_in_registry:"container-nasa_${project}-sds-pcm:${lowercase_pcm_branch}"
+#else
+#  sds -d ci add_job -b ${pcm_branch} --token https://${pcm_repo} s3
+#  sds -d ci build_job -b ${pcm_branch} https://${pcm_repo}
+#  sds -d ci remove_job -b ${pcm_branch} https://${pcm_repo}
+#fi
 
 #if [ "${delete_old_job_catalog}" = true ]; then
 #  python ~/mozart/ops/opera-pcm/job_accountability/create_job_accountability_catalog.py --delete_old_catalog

From 886cb6f927102c77b3100673ae8554bed3221283 Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Tue, 5 Apr 2022 13:11:52 -0700
Subject: [PATCH 45/48] Fix dev-e2e build error

---
 cluster_provisioning/dev-e2e/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster_provisioning/dev-e2e/main.tf b/cluster_provisioning/dev-e2e/main.tf
index c4e0d247a..c488498b6 100644
--- a/cluster_provisioning/dev-e2e/main.tf
+++ b/cluster_provisioning/dev-e2e/main.tf
@@ -75,7 +75,7 @@ module "common" {
   cluster_type                            = var.cluster_type
   data_download_timer_trigger_frequency   = var.data_download_timer_trigger_frequency
   data_query_timer_trigger_frequency      = var.data_query_timer_trigger_frequency
-#  obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
+  obs_acct_report_timer_trigger_frequency = var.obs_acct_report_timer_trigger_frequency
   rs_fwd_bucket_ingested_expiration       = var.rs_fwd_bucket_ingested_expiration
   dataset_bucket                          = var.dataset_bucket
   code_bucket                             = var.code_bucket

From 3fb613d95c5527ee8b92ea1560b52b19905332af Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Wed, 6 Apr 2022 10:47:55 -0700
Subject: [PATCH 46/48] Add job_accountability folder

---
 job_accountability/__init__.py                |  0
 job_accountability/catalog.py                 | 82 +++++++++++++++++++
 .../create_job_accountability_catalog.py      | 34 ++++++++
 job_accountability/es_connection.py           | 35 ++++++++
 .../es_mapping/job_mappings.json              | 50 +++++++++++
 5 files changed, 201 insertions(+)
 create mode 100644 job_accountability/__init__.py
 create mode 100644 job_accountability/catalog.py
 create mode 100755 job_accountability/create_job_accountability_catalog.py
 create mode 100644 job_accountability/es_connection.py
 create mode 100644 job_accountability/es_mapping/job_mappings.json

diff --git a/job_accountability/__init__.py b/job_accountability/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/job_accountability/catalog.py b/job_accountability/catalog.py
new file mode 100644
index 000000000..cfb2bc578
--- /dev/null
+++ b/job_accountability/catalog.py
@@ -0,0 +1,82 @@
+import os
+import json
+from hysds_commons.elasticsearch_utils import ElasticsearchUtility
+
+REFREC_ID = "refrec_id"
+ES_INDEX = "jobs_accountability_catalog"
+ES_TYPE = "job"
+HEADER = "header"
+
+
+class JobAccountabilityCatalog(ElasticsearchUtility):
+    """
+    Class to handle ingestion of All products past job-RRST_Accountability contents into ElasticSearch
+
+    https://github.com/hysds/hysds_commons/blob/develop/hysds_commons/elasticsearch_utils.py
+    ElasticsearchUtility methods
+        index_document
+        get_by_id
+        query
+        search
+        get_count
+        delete_by_id
+        update_document
+    """
+
+    def __add_mapping(self, index, mapping_type):
+        current_directory = os.path.dirname(os.path.abspath(__file__))
+        mappings_file = "{}_mappings.json".format(mapping_type)
+        mappings_file = os.path.join(current_directory, "es_mapping", mappings_file)
+
+        with open(mappings_file) as json_file:
+            self.es.indices.put_mapping(index=index, body=json.load(json_file))
+
+    def create_index(self, index=ES_INDEX, mapping_type=ES_TYPE, delete_old_index=False):
+        if delete_old_index is True:
+            self.es.indices.delete(index=index, ignore=404)
+            if self.logger:
+                self.logger.info("Deleted old index: {}".format(index))
+
+        self.es.indices.create(index=ES_INDEX)
+        if self.logger:
+            self.logger.info("Successfully created index: {}".format(index))
+
+        self.__add_mapping(index, mapping_type)  # Add mapping
+        if self.logger:
+            self.logger.info("Successfully add mapping to index {}".format(index))
+
+    def delete_index(self, index=ES_INDEX):
+        self.es.indices.delete(index=index, ignore=404)
+        if self.logger:
+            self.logger.info("Successfully deleted index: {}".format(index))
+
+    def post(self, records, header=None, index=ES_INDEX):
+        """
+        Post records into ElasticSearch.
+
+        :param records: A list of Pass Accountability records to ingest.
+        :param header: Header information associated with the records.
+        If present, they will get included with each record document posted.
+        :param index: Specify the index where the documents will get posted to.
+        :return:
+        """
+        for record in records:
+            if header:
+                record[HEADER] = header
+            else:
+                record[HEADER] = {}
+            if self.logger:
+                self.logger.info("record: {}".format(record))
+            self.__post_to_es(record, index)
+
+    def __post_to_es(self, document, index):
+        """
+        Posts the given document to ElasticSearch.
+
+        :param document: The document to ingest.
+        :return:
+        """
+        _id = document[REFREC_ID]
+        result = self.index_document(index=index, id=_id, body=document)
+        if self.logger:
+            self.logger.info("document indexed: {}".format(result))
diff --git a/job_accountability/create_job_accountability_catalog.py b/job_accountability/create_job_accountability_catalog.py
new file mode 100755
index 000000000..5add85b15
--- /dev/null
+++ b/job_accountability/create_job_accountability_catalog.py
@@ -0,0 +1,34 @@
+import logging
+import argparse
+
+from job_accountability.es_connection import get_job_accountability_connection
+
+
+logging.basicConfig(level=logging.INFO)  # Set up logging
+LOGGER = logging.getLogger(__name__)
+
+
+def get_parser():
+    """
+    Get a parser for this application
+    @return: parser to for this application
+    """
+    parser = argparse.ArgumentParser(description="Creates a Job Accountability catalog at a given endpoint.")
+    parser.add_argument("--delete_old_catalog", action='store_true',
+                        help="Indicate whether to delete the old COP Catalog if it exists.")
+    return parser
+
+
+if __name__ == "__main__":
+    """
+    Script to create the Pass Accountability Catalog and its mappings. Default is to create the
+    catalog in the GRQ ElasticSearch.
+    """
+    args = get_parser().parse_args()
+
+    delete_old_catalog = False
+    if args.delete_old_catalog:
+        delete_old_catalog = True
+
+    job_catalog = get_job_accountability_connection(LOGGER)
+    job_catalog.create_index(delete_old_index=delete_old_catalog)
diff --git a/job_accountability/es_connection.py b/job_accountability/es_connection.py
new file mode 100644
index 000000000..222446857
--- /dev/null
+++ b/job_accountability/es_connection.py
@@ -0,0 +1,35 @@
+from elasticsearch import RequestsHttpConnection
+from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
+
+from job_accountability.catalog import JobAccountabilityCatalog
+from hysds.celery import app
+
+PASS_ACC_CONN = None
+
+
+def get_job_accountability_connection(logger):
+    global PASS_ACC_CONN
+
+    if PASS_ACC_CONN is None:
+        aws_es = app.conf['GRQ_AWS_ES']
+        es_host = app.conf['GRQ_ES_HOST']
+        es_url = app.conf['GRQ_ES_URL']
+        region = app.conf['AWS_REGION']
+
+        if aws_es is True:
+            aws_auth = BotoAWSRequestsAuth(aws_host=es_host, aws_region=region, aws_service='es')
+            PASS_ACC_CONN = JobAccountabilityCatalog(
+                es_url=es_url,
+                logger=logger,
+                http_auth=aws_auth,
+                connection_class=RequestsHttpConnection,
+                use_ssl=True,
+                verify_certs=False,
+                ssl_show_warn=False,
+                timeout=30,
+                max_retries=10,
+                retry_on_timeout=True,
+            )
+        else:
+            PASS_ACC_CONN = JobAccountabilityCatalog(es_url, logger)
+    return PASS_ACC_CONN
diff --git a/job_accountability/es_mapping/job_mappings.json b/job_accountability/es_mapping/job_mappings.json
new file mode 100644
index 000000000..bf559cb8b
--- /dev/null
+++ b/job_accountability/es_mapping/job_mappings.json
@@ -0,0 +1,50 @@
+{
+  "properties": {
+    "ldf_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "ldf_state": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "vcid": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "ldf_state_config_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "nen_l_rrst_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "nen_l_rrst_job_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "l0a_l_rrst_pp_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "l0a_l_rrst_pp_job_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "l0a_l_rrst_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "l0a_l_rrst_job_id": {
+      "type": "text",
+      "copy_to": "text_fields"
+    },
+    "last_modified": {
+      "type": "date"
+    },
+    "text_fields": {
+      "type": "text"
+    }
+  }
+}

From 101f6a4edc93897667e1317c1a1079a0d0da07ec Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Thu, 7 Apr 2022 19:40:18 -0700
Subject: [PATCH 47/48] Change back terraform version >=0.13 until we upgrade
 terraform version on opera-pcm-ci

---
 .../dev-e2e-baseline-pge/versions.tf          | 22 +------------------
 .../dev-e2e-event-misfire/versions.tf         | 14 +-----------
 cluster_provisioning/dev-e2e-pge/versions.tf  | 22 +------------------
 cluster_provisioning/dev-releaser/versions.tf | 22 +------------------
 cluster_provisioning/dev/versions.tf          | 22 +------------------
 cluster_provisioning/modules/common/main.tf   |  3 +--
 .../modules/common/versions.tf                |  6 +++--
 7 files changed, 10 insertions(+), 101 deletions(-)
 mode change 100644 => 120000 cluster_provisioning/dev-e2e-baseline-pge/versions.tf
 mode change 100644 => 120000 cluster_provisioning/dev-e2e-event-misfire/versions.tf
 mode change 100644 => 120000 cluster_provisioning/dev-e2e-pge/versions.tf
 mode change 100644 => 120000 cluster_provisioning/dev-releaser/versions.tf
 mode change 100644 => 120000 cluster_provisioning/dev/versions.tf

diff --git a/cluster_provisioning/dev-e2e-baseline-pge/versions.tf b/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
deleted file mode 100644
index e465dbaa8..000000000
--- a/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-terraform {
-  required_version = ">= 1.0.11"
-
-  required_providers {
-    # TODO: remove this pin once this ticket is resolved:
-    #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    random = {
-      source = "hashicorp/random"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev-e2e-baseline-pge/versions.tf b/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev-e2e-baseline-pge/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/dev-e2e-event-misfire/versions.tf b/cluster_provisioning/dev-e2e-event-misfire/versions.tf
deleted file mode 100644
index 304e8bb72..000000000
--- a/cluster_provisioning/dev-e2e-event-misfire/versions.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-
-terraform {
-  required_version = ">= 1.0.11"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev-e2e-event-misfire/versions.tf b/cluster_provisioning/dev-e2e-event-misfire/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev-e2e-event-misfire/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/dev-e2e-pge/versions.tf b/cluster_provisioning/dev-e2e-pge/versions.tf
deleted file mode 100644
index e465dbaa8..000000000
--- a/cluster_provisioning/dev-e2e-pge/versions.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-terraform {
-  required_version = ">= 1.0.11"
-
-  required_providers {
-    # TODO: remove this pin once this ticket is resolved:
-    #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    random = {
-      source = "hashicorp/random"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev-e2e-pge/versions.tf b/cluster_provisioning/dev-e2e-pge/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev-e2e-pge/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/dev-releaser/versions.tf b/cluster_provisioning/dev-releaser/versions.tf
deleted file mode 100644
index e465dbaa8..000000000
--- a/cluster_provisioning/dev-releaser/versions.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-terraform {
-  required_version = ">= 1.0.11"
-
-  required_providers {
-    # TODO: remove this pin once this ticket is resolved:
-    #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    random = {
-      source = "hashicorp/random"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev-releaser/versions.tf b/cluster_provisioning/dev-releaser/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev-releaser/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/dev/versions.tf b/cluster_provisioning/dev/versions.tf
deleted file mode 100644
index e465dbaa8..000000000
--- a/cluster_provisioning/dev/versions.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-terraform {
-  required_version = ">= 1.0.11"
-
-  required_providers {
-    # TODO: remove this pin once this ticket is resolved:
-    #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
-    aws = {
-      source  = "hashicorp/aws"
-      version = "2.70.1"
-    }
-    null = {
-      source = "hashicorp/null"
-    }
-    random = {
-      source = "hashicorp/random"
-    }
-    template = {
-      source = "hashicorp/template"
-    }
-  }
-}
diff --git a/cluster_provisioning/dev/versions.tf b/cluster_provisioning/dev/versions.tf
new file mode 120000
index 000000000..5c7afe986
--- /dev/null
+++ b/cluster_provisioning/dev/versions.tf
@@ -0,0 +1 @@
+../modules/common/versions.tf
\ No newline at end of file
diff --git a/cluster_provisioning/modules/common/main.tf b/cluster_provisioning/modules/common/main.tf
index 0ec602bf3..84d17849d 100644
--- a/cluster_provisioning/modules/common/main.tf
+++ b/cluster_provisioning/modules/common/main.tf
@@ -1950,7 +1950,6 @@ resource "aws_lambda_permission" "event-misfire_lambda" {
 #    variables = {
 #      "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
 #      "JOB_QUEUE": "${var.project}-job_worker-small",
-#      #"JOB_QUEUE": "opera-job_worker-small",
 #      "JOB_TYPE": local.accountability_report_job_type,
 #      "JOB_RELEASE": var.pcm_branch,
 #      "REPORT_NAME": "ObservationAccountabilityReport",
@@ -2008,7 +2007,7 @@ resource "aws_lambda_function" "data_subscriber_download_timer" {
   environment {
     variables = {
       "MOZART_URL": "https://${aws_instance.mozart.private_ip}/mozart",
-      "JOB_QUEUE": "opera-job_worker-small",
+	  "JOB_QUEUE": "${var.project}-job_worker-small",
       "JOB_TYPE": local.data_download_job_type,
       "JOB_RELEASE": var.pcm_branch,
       "ISL_BUCKET_NAME": local.isl_bucket,
diff --git a/cluster_provisioning/modules/common/versions.tf b/cluster_provisioning/modules/common/versions.tf
index e465dbaa8..17e26bd22 100644
--- a/cluster_provisioning/modules/common/versions.tf
+++ b/cluster_provisioning/modules/common/versions.tf
@@ -1,12 +1,14 @@
 terraform {
-  required_version = ">= 1.0.11"
+  #required_version = ">= 1.0.11"
+  required_version = ">= 0.13"
 
   required_providers {
     # TODO: remove this pin once this ticket is resolved:
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-      version = "2.70.1"
+	  version = "2.63"
+#      version = "2.70.1"
     }
     null = {
       source = "hashicorp/null"

From 8807ca9fa3b331a0ea745ecce94ae58dce0b922c Mon Sep 17 00:00:00 2001
From: hyunlee <Hyun.H.Lee@jpl.nasa.gov>
Date: Fri, 8 Apr 2022 09:22:05 -0700
Subject: [PATCH 48/48] Set pge_release to 1.0.0-er.3.0 in variables.tf

---
 .../dev-e2e-baseline-pge/variables.tf         |  4 +-
 .../dev-e2e-event-misfire/variables.tf        |  4 +-
 cluster_provisioning/dev-e2e-pge/variables.tf |  4 +-
 cluster_provisioning/dev-e2e/variables.tf     |  4 +-
 .../dev-restore-snapshot/variables.tf         |  4 +-
 cluster_provisioning/dev/variables.tf         |  4 +-
 .../ebs-snapshot/variables.tf                 |  4 +-
 cluster_provisioning/int/override.tf          | 62 ++++++++++++-------
 8 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
index 9b089f442..a2c727ed0 100644
--- a/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-baseline-pge/variables.tf
@@ -354,11 +354,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220318-R1.0.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "R1.0.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/dev-e2e-event-misfire/variables.tf b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
index 54cf0d55b..1f5a6e9c8 100644
--- a/cluster_provisioning/dev-e2e-event-misfire/variables.tf
+++ b/cluster_provisioning/dev-e2e-event-misfire/variables.tf
@@ -346,11 +346,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220318-R1.0.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "R1.0.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/dev-e2e-pge/variables.tf b/cluster_provisioning/dev-e2e-pge/variables.tf
index a2fcc0a1d..fb131c91a 100644
--- a/cluster_provisioning/dev-e2e-pge/variables.tf
+++ b/cluster_provisioning/dev-e2e-pge/variables.tf
@@ -392,11 +392,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220318-R1.0.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "R1.0.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/dev-e2e/variables.tf b/cluster_provisioning/dev-e2e/variables.tf
index 48e8dc7fd..4514feef7 100644
--- a/cluster_provisioning/dev-e2e/variables.tf
+++ b/cluster_provisioning/dev-e2e/variables.tf
@@ -346,11 +346,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220318-R1.0.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "R1.0.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/dev-restore-snapshot/variables.tf b/cluster_provisioning/dev-restore-snapshot/variables.tf
index 1eef21a04..2cf0ebd6d 100644
--- a/cluster_provisioning/dev-restore-snapshot/variables.tf
+++ b/cluster_provisioning/dev-restore-snapshot/variables.tf
@@ -346,11 +346,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220208-develop-ER2.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/dev/variables.tf b/cluster_provisioning/dev/variables.tf
index fcc4ae97f..ef4be022f 100644
--- a/cluster_provisioning/dev/variables.tf
+++ b/cluster_provisioning/dev/variables.tf
@@ -346,11 +346,11 @@ variable "lambda_log_retention_in_days" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220208-develop-ER2.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "crid" {
diff --git a/cluster_provisioning/ebs-snapshot/variables.tf b/cluster_provisioning/ebs-snapshot/variables.tf
index 93b61be7f..98e4005fa 100644
--- a/cluster_provisioning/ebs-snapshot/variables.tf
+++ b/cluster_provisioning/ebs-snapshot/variables.tf
@@ -25,11 +25,11 @@ variable "logstash_release" {
 }
 
 variable "pge_snapshots_date" {
-  default = "20220208-develop-ER2.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "v1.0.0-er.3.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "private_key_file" {
diff --git a/cluster_provisioning/int/override.tf b/cluster_provisioning/int/override.tf
index abd6c4605..b64204cd5 100644
--- a/cluster_provisioning/int/override.tf
+++ b/cluster_provisioning/int/override.tf
@@ -7,7 +7,7 @@
 
 ##### Environments #######
 variable "aws_account_id" {
-  default = "399787141461"
+  default = "337765570207"
 }
 
 variable "venue" {
@@ -20,6 +20,7 @@ variable "environment" {
 
 variable "counter" {
   default = "fwd"
+  #default = "pop1"
 }
 
 variable "crid" {
@@ -46,19 +47,27 @@ variable "cluster_type" {
 
 ###### Security  ########
 variable "verdi_security_group_id" {
-  default = "sg-0c6342d8c098f1491"
+  # fwd security group
+  default = "sg-0869719f04e735bd6"
+  # pop1 security group
+  #default = "sg-045ff9d3d16a65ba4"
 }
 
 variable "cluster_security_group_id" {
-  default = "sg-06cad001f63628a45"
+  # fwd security group
+  default = "sg-039db67f56d1b12f0"
+  # pop1 security group
+  #default = "sg-06bf23a69b4d83f66"
 }
 
 variable "private_key_file" {
-  default = "~/.ssh/operasds-int-cluster-1.pem"
+  default = "~/.ssh/operasds-int-cluster-fwd.pem"
+  #default = "~/.ssh/operasds-int-cluster-pop1.pem"
 }
 
 variable "keypair_name" {
-  default = "operasds-int-cluster-1"
+  default = "operasds-int-cluster-fwd"
+#  default = "operasds-int-cluster-pop1"
 }
 
 variable "ops_password" {
@@ -78,45 +87,52 @@ variable "profile" {
 
 ####### Subnet ###########
 variable "subnet_id" {
-  default = "subnet-000eb551ad06392c7"
+  default = "subnet-0644c172bef1d690e"
 }
 
 ####### VPC #########
 variable "lambda_vpc" {
-  default = "vpc-02676637ea26098a7"
+  default = "vpc-07cd74102c0dfd9ab"
 }
 
 variable "asg_vpc" {
-  default = "vpc-02676637ea26098a7"
+  default = "vpc-c1e0dab9"
 }
 
 ##### Bucket Names #########
 variable "docker_registry_bucket" {
   default = "opera-int-cc-fwd"
+  #default = "opera-int-cc-pop1"
 }
 
 variable "dataset_bucket" {
   default = "opera-int-rs-fwd"
+  #default = "opera-int-rs-pop1"
 }
 
 variable "code_bucket" {
   default = "opera-int-cc-fwd"
+  #default = "opera-int-cc-pop1"
 }
 
 variable "lts_bucket" {
   default = "opera-int-lts-fwd"
+  #default = "opera-int-lts-pop1"
 }
 
 variable "triage_bucket" {
   default = "opera-int-triage-fwd"
+  #default = "opera-int-triage-pop1"
 }
 
 variable "isl_bucket" {
   default = "opera-int-isl-fwd"
+  #default = "opera-int-isl-pop1"
 }
 
 variable "osl_bucket" {
   default = "opera-int-osl-fwd"
+  #default = "opera-int-osl-pop1"
 }
 
 variable "es_snapshot_bucket" {
@@ -132,50 +148,50 @@ variable "amis" {
   type = map(string)
   default = {
 	# HySDS v4.0.1-beta8-oraclelinux
-	mozart    = "ami-02fcd254c71ff0fa0"  # opera dev mozart - ol8
-    metrics   = "ami-0a54a14946e0bb52f"  # opera dev metrics - ol8
-    grq       = "ami-0a11c7d42e24fe7d5"  # opera dev grq - ol8
-    factotum  = "ami-0ce5e6a66b7732993"  # opera dev factotum - ol8
-    ci        = "ami-0caed57c920d65ea8"  # OL8 All-project verdi v4.11
-    autoscale = "ami-0caed57c920d65ea8"  # OL8 All-project verdi v4.11
+    mozart    = "ami-014ef2d57c80be48c"  # opera int mozart - ol8
+    metrics   = ""  # opera int metrics - ol8
+    grq       = ""  # opera int grq - ol8
+    factotum  = ""  # opera int factotum - ol8
+    ci        = "ami-0e4564f8e531e816f"  # OL8 All-project verdi v4.12
+    autoscale = "ami-0e4564f8e531e816f"  # OL8 All-project verdi v4.12
   }
 }
 
 ####### Release Branches #############
 variable "pge_snapshots_date" {
-  default = "20220208-develop-ER2.0"
+  default = "20220401-1.0.0-er.3.0"
 }
 
 variable "pge_release" {
-  default = "1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "hysds_release" {
-  default = "v4.0.1-beta.2"
+  default = "v4.0.1-beta.8-oraclelinux"
 }
 
 variable "lambda_package_release" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "pcm_commons_branch" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "pcm_branch" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "product_delivery_branch" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "bach_api_branch" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 variable "bach_ui_branch" {
-  default = "v1.0.0-er.2.0"
+  default = "1.0.0-er.3.0"
 }
 
 ###### Roles ########