feat(2767): Add stage setup and teardown jobIds (#570)

index.js

@@ -11,6 +11,7 @@ const JobFactory = require('./lib/jobFactory');
 const PipelineFactory = require('./lib/pipelineFactory');
 const SecretFactory = require('./lib/secretFactory');
 const StageFactory = require('./lib/stageFactory');
+const StageBuildFactory = require('./lib/stageBuildFactory');
 const StepFactory = require('./lib/stepFactory');
 const TemplateFactory = require('./lib/templateFactory');
 const TemplateTagFactory = require('./lib/templateTagFactory');
@@ -35,6 +36,7 @@ module.exports = {
     PipelineFactory,
     SecretFactory,
     StageFactory,
+    StageBuildFactory,
     StepFactory,
     TemplateFactory,
     TemplateTagFactory,

lib/event.js

@@ -15,8 +15,25 @@ class EventModel extends BaseModel {
         super('event', config);
     }
 
+    /**
+     * Return stage builds that belong to this event
+     * @method getStageBuilds
+     * @return {Promise}  Resolves to an array of stage builds
+     */
+    async getStageBuilds() {
+        // Lazy load factory dependency to prevent circular dependency issues
+        // https://nodejs.org/api/modules.html#modules_cycles
+        /* eslint-disable global-require */
+        const StageBuildFactory = require('./stageBuildFactory');
+        /* eslint-enable global-require */
+        const stageBuildFactory = StageBuildFactory.getInstance();
+
+        return stageBuildFactory.list({ params: { eventId: this.id } });
+    }
+
     /**
      * Return builds that belong to this event
+     * @method getBuilds
      * @param  {String}   [config.startTime]     Search for builds after this startTime
      * @param  {String}   [config.endTime]       Search for builds before this endTime
      * @param  {String}   [config.sort]          Ascending or descending

lib/helper.js

@@ -10,6 +10,7 @@ const DEFAULT_KEY = 'default';
 const EXECUTOR_ANNOTATION = 'screwdriver.cd/executor';
 const EXECUTOR_ANNOTATION_BETA = 'beta.screwdriver.cd/executor';
 const SCM_ORG_REGEX = /^([^/]+)\/.*/;
+const STAGE_PREFIX = 'stage@';
 
 /**
  * Get the value of the annotation that matches name
@@ -398,12 +399,23 @@ async function getBookendKey({ buildClusterName, annotations, pipeline, provider
     };
 }
 
+/**
+ * Returns full stage name with correct formatting and setup or teardown suffix (e.g. stage@deploy:setup)
+ * @param  {String} stageName               Stage name
+ * @param  {String} type                    Type of stage job, either 'setup' or 'teardown'
+ * @return {String}                         Full stage name
+ */
+function getFullStageJobName({ stageName, jobName }) {
+    return `${STAGE_PREFIX}${stageName}:${jobName}`;
+}
+
 module.exports = {
     getAnnotations,
     convertToBool,
     parseTemplateConfigName,
     getAllRecords,
     getBuildClusterName,
     getToken,
-    getBookendKey
+    getBookendKey,
+    getFullStageJobName
 };

lib/pipeline.js

@@ -22,7 +22,7 @@ const MAX_EVENT_DELETE_COUNT = 100;
 const DEFAULT_PAGE = 1;
 const SCM_NO_ACCESS_STATUSES = [401, 404];
 
-const { getAllRecords, getBuildClusterName } = require('./helper');
+const { getAllRecords, getBuildClusterName, getFullStageJobName } = require('./helper');
 
 const JOB_CHUNK_SIZE = process.env.JOBS_PARALLEL_COUNT || 5;
 const SD_API_URI = process.env.URI;
@@ -418,6 +418,7 @@ class PipelineModel extends BaseModel {
 
     /**
      * archive closed PR jobs
+     * @method _archiveClosePRs
      * @param {Array}   existingPrJobs List pipeline's existing pull request jobs (excludes already archived jobs for closed PRs)
      * @param {Array}   openedPRs    List of opened PRs coming from SCM
      * @param {Promise}
@@ -852,6 +853,110 @@ class PipelineModel extends BaseModel {
         return Promise.allSettled([...toCreateOrUpdate, ...toDeactivate]);
     }
 
+    /**
+     * Converts the simplified stages into a more consistent format
+     *
+     * This is because the user can provide the stage information as:
+     *  - {"name": { "jobs": ["job1", "job2", "job3"], "description": "Description" },
+     *     { "name2": { "jobs": ["job4", "job5"] } }
+     *
+     * We will convert it to a more standard format:
+     *  - [{ "name": "name", "jobs": [1, 2, 3], "pipelineId": 123, "description": "value" },
+     *     { "name": "name2", "jobs": [4, 5], "pipelineId": 123 }]
+     * @method convertStages
+     * @param  {Object}     config              config
+     * @param  {Object}     config.pipeline     Pipeline
+     * @param  {Object}     config.stages       Pipeline stages
+     * @return {Array}                New array with stages after up-converting
+     */
+    async _convertStages({ pipelineId, stages, pipelineJobs }) {
+        const newStages = [];
+
+        // Convert stages from object to array of objects
+        Object.entries(stages).forEach(([key, value]) => {
+            const newStage = {
+                name: key,
+                pipelineId,
+                ...value
+            };
+
+            // Convert the jobNames to jobIds
+            newStage.jobIds = value.jobs.map(jobName => {
+                return pipelineJobs.find(j => j.name === jobName).id;
+            });
+
+            delete newStage.jobs; // extra field from yaml parser
+
+            // Check for setup and teardown
+            const setupJobName = getFullStageJobName({ stageName: key, jobName: 'setup' });
+            const teardownJobName = getFullStageJobName({ stageName: key, jobName: 'teardown' });
+
+            newStage.setup = pipelineJobs.find(j => j.name === setupJobName).id;
+            newStage.teardown = pipelineJobs.find(j => j.name === teardownJobName).id;
+
+            newStages.push(newStage);
+        });
+
+        return newStages;
+    }
+
+    /**
+     * Sync stages
+     * 1. Convert new stages into correct format, prepopulate with jobIds
+     * 2.a. Create stages if they are defined and were not already in the database
+     * 2.b. Update existing stages with the new configuration
+     * 2.c. Archive existing stages if they no longer exist in the configuration
+     * @method _createOrUpdateStages
+     * @param  {Object}     config              config
+     * @param  {Object}     config.pipeline     Pipeline
+     * @return {Promise}
+     */
+    async _createOrUpdateStages({ parsedConfig, pipelineId, stageFactory, pipelineJobs }) {
+        // Get new stages
+        const stages = parsedConfig.stages || {};
+
+        // list stage names from this pipeline that already exist
+        const existingStages = await stageFactory.list({ params: { pipelineId } });
+        const existingStageNames = existingStages.map(stage => stage.name);
+        // Format new stage data
+        const convertedStages = await this._convertStages({ pipelineId, stages, pipelineJobs });
+        const convertedStageNames = convertedStages.map(stage => stage.name);
+
+        const stagesToUpdate = convertedStages.filter(stage => existingStageNames.includes(stage.name));
+        const stagesToCreate = convertedStages.filter(stage => !existingStageNames.includes(stage.name));
+        const stagesToArchive = existingStages.filter(stage => !convertedStageNames.includes(stage.name));
+        const processed = [];
+
+        // Archive outdated stages
+        stagesToArchive.forEach(stage => {
+            const existingStage = existingStages.find(s => s.name === stage.name);
+
+            existingStage.archived = true;
+
+            logger.info(`Archiving stage:${JSON.stringify(stage)} for pipelineId:${pipelineId}.`);
+            processed.push(existingStage.update());
+        });
+
+        // Update existing stages
+        stagesToUpdate.forEach(stage => {
+            const existingStage = existingStages.find(s => s.name === stage.name);
+
+            Object.assign(existingStage, stage);
+            existingStage.archived = false;
+
+            logger.info(`Updating stage:${JSON.stringify(stage)} for pipelineId:${pipelineId}.`);
+            processed.push(existingStage.update());
+        });
+
+        // Create new stages
+        stagesToCreate.forEach(stage => {
+            logger.info(`Creating stage:${JSON.stringify(stage)} for pipelineId:${pipelineId}.`);
+            processed.push(stageFactory.create(stage));
+        });
+
+        return Promise.all(processed);
+    }
+
     /**
      * Sync the pipeline by looking up screwdriver.yaml
      * Create, update, or disable jobs if necessary.
@@ -867,7 +972,14 @@ class PipelineModel extends BaseModel {
         /* eslint-disable global-require */
         const JobFactory = require('./jobFactory');
         /* eslint-enable global-require */
-        const factory = JobFactory.getInstance();
+        const jobFactory = JobFactory.getInstance();
+
+        // Lazy load factory dependency to prevent circular dependency issues
+        // https://nodejs.org/api/modules.html#modules_cycles
+        /* eslint-disable global-require */
+        const StageFactory = require('./stageFactory');
+        /* eslint-enable global-require */
+        const stageFactory = StageFactory.getInstance();
 
         // get the pipeline configuration
         const parsedConfig = await this.getConfiguration({ ref });
@@ -970,7 +1082,7 @@ class PipelineModel extends BaseModel {
 
                     // If the job has not been processed, create it (new jobs)
                     if (!jobsProcessed.includes(jobName)) {
-                        updatedJobs.push(await factory.create(jobConfig));
+                        updatedJobs.push(await jobFactory.create(jobConfig));
 
                         await syncExternalTriggers({
                             pipelineId,
@@ -982,6 +1094,14 @@ class PipelineModel extends BaseModel {
             );
         }
 
+        // Sync stages
+        await this._createOrUpdateStages({
+            parsedConfig,
+            pipelineId,
+            stageFactory,
+            pipelineJobs: updatedJobs
+        });
+
         const { nodes } = this.workflowGraph;
 
         // Add jobId to workflowGraph.nodes

lib/stage.js

@@ -9,9 +9,11 @@ class StageModel extends BaseModel {
      * @param  {Object}    config
      * @param  {Object}    config.datastore     Object that will perform operations on the datastore
      * @param  {String}    [config.description] Stage description
-     * @param  {Array}     [config.jobIds=[]]   Job Ids that belong to this stage
+     * @param  {Array}     [config.jobIds=[]]   Job IDs that belong to this stage
      * @param  {String}    config.name          Name of the stage
      * @param  {Number}    config.pipelineId    Pipeline the stage belongs to
+     * @param  {Array}     [config.setup]       Setup job IDs
+     * @param  {Array}     [config.teardown]    Teardown job IDs
      */
     constructor(config) {
         super('stage', config);

lib/stageBuild.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const BaseModel = require('./base');
+
+class StageBuildModel extends BaseModel {
+    /**
+     * Construct a StageBuildModel object
+     * @method constructor
+     * @param  {Object}    config
+     * @param  {Object}    config.datastore     Object that will perform operations on the datastore
+     * @param  {Number}    config.eventId       Event ID
+     * @param  {Number}    config.stageId       Stage ID
+     * @param  {String}    config.status        Stage build status
+     */
+    constructor(config) {
+        super('stageBuild', config);
+    }
+}
+
+module.exports = StageBuildModel;

lib/stageBuildFactory.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const BaseFactory = require('./baseFactory');
+const StageBuild = require('./stageBuild');
+let instance;
+
+class StageBuildFactory extends BaseFactory {
+    /**
+     * Construct a StageBuildFactory object
+     * @method constructor
+     * @param  {Object}     config
+     * @param  {Datastore}  config.datastore     Object that will perform operations on the datastore
+     */
+    constructor(config) {
+        super('stageBuild', config);
+    }
+
+    /**
+     * Instantiate a StageBuild class
+     * @method createClass
+     * @param  {Object}     config               StageBuild data
+     * @return {StageBuild}
+     */
+    createClass(config) {
+        return new StageBuild(config);
+    }
+
+    /**
+     * Create a StageBuild model
+     * @param {Object}    config
+     * @param {Number}    config.eventId       Event ID
+     * @param {Number}    config.stageId       Stage ID
+     * @param {Object}    config.workflowGraph Stage workflowGraph
+     * @memberof StageBuildFactory
+     */
+    create(config) {
+        return super.create(config);
+    }
+
+    /**
+     * Get an instance of the StageBuildFactory
+     * @method getInstance
+     * @param  {Object}     config
+     * @param  {Datastore}  config.datastore
+     * @return {StageBuildFactory}
+     */
+    static getInstance(config) {
+        instance = BaseFactory.getInstance(StageBuildFactory, instance, config);
+
+        return instance;
+    }
+}
+
+module.exports = StageBuildFactory;

lib/stageFactory.js

@@ -27,14 +27,18 @@ class StageFactory extends BaseFactory {
 
     /**
      * Create a Stage model
-     * @param {Object}      config
-     * @param {String}      [config.description] Stage description
-     * @param {Array}       [config.jobIds=[]]   Job Ids that belong to this stage
-     * @param {String}      config.name          Name of the stage
-     * @param {String}      config.pipelineId    Pipeline the stage belongs to
+     * @param {Object}  config
+     * @param {String}  [config.description] Stage description
+     * @param {Array}   [config.jobIds=[]]   Job IDs that belong to this stage
+     * @param {String}  config.name          Name of the stage
+     * @param {Number}  config.pipelineId    Pipeline the stage belongs to
+     * @param {Array}   [config.setup]       Setup job IDs
+     * @param {Array}   [config.teardown]    Teardown job IDs
      * @memberof StageFactory
      */
     create(config) {
+        config.archived = false;
+
         return super.create(config);
     }
 

package.json

@@ -57,9 +57,9 @@
     "docker-parse-image": "^3.0.1",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
-    "screwdriver-config-parser": "^8.0.0",
-    "screwdriver-data-schema": "^22.6.1",
+    "screwdriver-config-parser": "^8.0.3",
+    "screwdriver-data-schema": "^22.9.7",
     "screwdriver-logger": "^2.0.0",
-    "screwdriver-workflow-parser": "^4.0.0"
+    "screwdriver-workflow-parser": "^4.1.0"
   }
 }