Skip to content

Commit

Permalink
Merge pull request #290 from saagie/feat-export-import-env-var
Browse files Browse the repository at this point in the history 
Add export import variable for v2
  • Loading branch information
@medamineziraoui
medamineziraoui authored Aug 12, 2020
2 parents c2c6500 + c8e578c commit 1caab16
Show file tree
Hide file tree
Showing 22 changed files with 785 additions and 38 deletions.
8 changes: 8 additions & 0 deletions src/main/groovy/io/saagie/plugin/dataops/DataOpsExtension.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.saagie.plugin.dataops

import groovy.transform.TypeChecked
import io.saagie.plugin.dataops.models.EnvVariable
import io.saagie.plugin.dataops.models.Export
import io.saagie.plugin.dataops.models.ImportJob
import io.saagie.plugin.dataops.models.Job
Expand Down Expand Up @@ -45,6 +46,8 @@ class DataOpsExtension {

Export exportArtifacts = new Export()

EnvVariable env = new EnvVariable()

// ====== Closures to create a proper DSL
Object server( Closure closure ) {
server = new Server()
Expand Down Expand Up @@ -76,6 +79,11 @@ class DataOpsExtension {
pipeline.with( closure )
}

Object env( Closure closure ) {
env = new EnvVariable()
env.with( closure )
}

Object pipelineVersion( Closure closure ) {
pipelineVersion = new PipelineVersion()
pipelineVersion.with( closure )
Expand Down
212 changes: 201 additions & 11 deletions src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClient.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@ package io.saagie.plugin.dataops.clients
import groovy.json.JsonOutput
import groovy.json.JsonSlurper
import io.saagie.plugin.dataops.DataOpsExtension
import io.saagie.plugin.dataops.models.EnvVarScopeTypeEnum
import io.saagie.plugin.dataops.models.ExportJobs
import io.saagie.plugin.dataops.models.ExportPipeline
import io.saagie.plugin.dataops.models.ExportVariables
import io.saagie.plugin.dataops.models.Job
import io.saagie.plugin.dataops.models.JobVersion
import io.saagie.plugin.dataops.models.PipelineVersion
import io.saagie.plugin.dataops.models.ResponseStatusEnum
import io.saagie.plugin.dataops.models.Server
import io.saagie.plugin.dataops.models.Pipeline
import io.saagie.plugin.dataops.models.PipelineVersion
import io.saagie.plugin.dataops.tasks.projects.enums.JobV1Category
import io.saagie.plugin.dataops.tasks.projects.enums.JobV1Type
import io.saagie.plugin.dataops.tasks.service.TechnologyService
import io.saagie.plugin.dataops.tasks.service.VariableService
import io.saagie.plugin.dataops.tasks.service.exportTask.ExportContainer
import io.saagie.plugin.dataops.tasks.service.importTask.ImportJobService
import io.saagie.plugin.dataops.tasks.service.importTask.ImportPipelineService
import io.saagie.plugin.dataops.tasks.service.importTask.ImportVariableService
import io.saagie.plugin.dataops.utils.HttpClientBuilder
import io.saagie.plugin.dataops.utils.SaagieUtils
import io.saagie.plugin.dataops.utils.ZipUtils
Expand Down Expand Up @@ -725,6 +729,34 @@ class SaagieClient {
}, 'Unknown error in task : deleteProjectJob', 'Function: deleteProjectJob')
}

def saveEnvironmentVariable( environmentVariable ) {
logger.info('saveEnvironmentVariable')
def requestSaveEnvironmentVariable = null
def isProjectRequest = environmentVariable.scope.equals(EnvVarScopeTypeEnum.project.name().toUpperCase())
if (isProjectRequest) {
requestSaveEnvironmentVariable = saagieUtils.saveProjectEnvironmentVariable(environmentVariable)
} else {
requestSaveEnvironmentVariable = saagieUtils.saveGlobalEnvironmentVariable(environmentVariable)
}
tryCatchClosure({
client.newCall(requestSaveEnvironmentVariable).execute().withCloseable { response ->
handleErrors(response)
String responseBody = response.body().string()
def parsedResult = slurper.parseText(responseBody)

if (parsedResult.errors) {
def message = "Something went wrong when saving environment variable: $responseBody"
logger.error(message)
throw new GradleException(message)
} else {
logger.debug("saved environment variable : $responseBody")
return parsedResult.data.saveEnvironmentVariable

}
}
}, 'Unknown error in task : importJob', 'Function: saveEnvironmentVariable')
}

String stopPipelineInstance() {
logger.info('Starting stopPipelineInstance task')
checkRequiredConfig(!configuration?.pipelineinstance?.id)
Expand Down Expand Up @@ -884,7 +916,9 @@ class SaagieClient {
exportJobs = getListJobAndJobVersionsFromConfigV1(listJobsByNameAndIdFromV1)
}

return export(exportPipelines, exportJobs, listJobsByNameAndIdFromV1, true)
ExportVariables[] exportVariables = []

return export(exportPipelines, exportJobs, exportVariables, listJobsByNameAndIdFromV1, true)
}

String exportArtifacts() {
Expand All @@ -903,11 +937,17 @@ class SaagieClient {
exportJobs = getListJobAndJobVersionsFromConfig()
}

return export(exportPipelines, exportJobs)
ExportVariables[] exportVariables = []
def validateVariableConfigurationParams = configuration.env.include_all_var || (configuration.env.name && configuration.env.name.size())
if (validateVariableConfigurationParams) {
exportVariables = getListVariablesFromConfig()
}

return export(exportPipelines, exportJobs, exportVariables)
}


String export( ExportPipeline[] exportPipelines, ExportJobs[] exportJobs, listJobWithNameAndIdV1 = null, isV1 = false ) {
String export( ExportPipeline[] exportPipelines, ExportJobs[] exportJobs, ExportVariables[] exportVariables, listJobWithNameAndIdV1 = null, isV1 = false ) {

ExportContainer exportContainer = [configuration]
boolean customDirectoryExist = false
Expand All @@ -928,12 +968,13 @@ class SaagieClient {
def listJobs = null
if (isV1) {
listJobs = listJobWithNameAndIdV1
} else {
} else if (exportPipelines.size() > 0 || exportJobs.size() > 0) {
listJobs = getProjectJobsByNameAndId()
}
def inputDirectoryToZip = tempJobDirectory.getAbsolutePath() + File.separator + exportContainer.fileName
folder.exportJobList = exportJobs
folder.exportPipelineList = exportPipelines
folder.exportVariableList = exportVariables
folder.jobList = listJobs
folder.generateFolderFromParams()
ZippingFolder zippingFolder = [exportContainer.exportConfigPath, inputDirectoryToZip, customDirectoryExist]
Expand Down Expand Up @@ -1424,6 +1465,97 @@ class SaagieClient {

}

ExportVariables[] getListVariablesFromConfig() {
logger.info('Starting getting environment variables from configuration for v2 ... ')
checkRequiredConfig(
checkIfEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject() ||
checkIfEnvScopeIsProjectButNoProjectIdProvided() ||
checkIfOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty()
)
def listVariables = []
tryCatchClosure({
Request variablesListRequest = null
if (configuration.env.scope.equals(EnvVarScopeTypeEnum.global.name())) {
variablesListRequest = saagieUtils.getGlobalEnvironmentVariables()
} else {
variablesListRequest = saagieUtils.getProjectEnvironmentVariables(configuration.project.id)
}
client.newCall(variablesListRequest).execute().withCloseable { responseVariableList ->
handleErrors(responseVariableList)
String responseBodyForVariableList = responseVariableList.body().string()
logger.debug("variable list :", responseBodyForVariableList)
def parsedResultForVariableList = slurper.parseText(responseBodyForVariableList)
if (configuration.env.scope.equals(EnvVarScopeTypeEnum.global.name()) && parsedResultForVariableList.data?.globalEnvironmentVariables) {
listVariables = parsedResultForVariableList.data.globalEnvironmentVariables
} else if (parsedResultForVariableList.data?.projectEnvironmentVariables) {
listVariables = parsedResultForVariableList.data.projectEnvironmentVariables
}
}
if (listVariables.size().equals(0)) {
return null
}

if (configuration.env.scope.equals(EnvVarScopeTypeEnum.project.name())) {
listVariables = listVariables.findAll {
it.scope?.equals(EnvVarScopeTypeEnum.project.name().toUpperCase())
}.collect {
it
}
}

if (!configuration.env.include_all_var && configuration.env.name && configuration.env.name.size() > 0) {
listVariables = listVariables.findAll {
configuration.env.name.contains(it.name)
}.collect {
it
}
}


if (!configuration.env.include_all_var) {
configuration.env.name.forEach {
def foundName = listVariables.find { var -> var.name?.equals(it) }
if (!foundName) {
throw new GradleException("Didn't find variable name: ${it} in the required environment variables list for scope ${configuration.env.scope.equals(EnvVarScopeTypeEnum.project.name()) ? EnvVarScopeTypeEnum.project.name().toString() : EnvVarScopeTypeEnum.global.name().toString()}")
}
}
}

def exportVariables = []

listVariables.forEach {
ExportVariables newExportVariable = []
newExportVariable.variableEnvironmentDTO.setVariableDetailValuesFromData(it)
exportVariables.add(newExportVariable)
}

return exportVariables

}, 'Error in getListVariablesFromConfig', 'getGlobalEnvironmentVariables | getProjectEnvironmentVariables Request') as ExportVariables[]

}

def checkIfEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject() {
def conditionForEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject = (!configuration.env.scope || (!configuration.env.scope.equals(EnvVarScopeTypeEnum.global.name()) && !configuration.env.scope.equals(EnvVarScopeTypeEnum.project.name())))
logger.debug("result for checkIfEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject")
logger.debug(conditionForEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject as String)
return conditionForEnvHaveNotScopeOrScopeDifferentFromGlobalAndProject
}

def checkIfEnvScopeIsProjectButNoProjectIdProvided() {
def conditionForScopeIsProjectButNoProjectIdProvided = (configuration.env.scope.equals(EnvVarScopeTypeEnum.project.name()) && configuration.project.id.equals(null))
logger.debug("result for checkIfEnvScopeIsProjectButNoProjectIdProvided")
logger.debug(conditionForScopeIsProjectButNoProjectIdProvided as String)
return conditionForScopeIsProjectButNoProjectIdProvided
}

def checkIfOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty() {
def conditionForOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty = (!configuration.env.include_all_var && (!configuration.env.name || !configuration.env.name.size()))
logger.debug("result for checkIfOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty")
logger.debug(conditionForOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty as String)
return conditionForOptionIncludeAllVarIsNotSetAndEnvNameAreEmpty
}

String updateProject() {
logger.info('Starting projectsUpdate task')

Expand Down Expand Up @@ -1483,13 +1615,15 @@ class SaagieClient {
def exportedArtifactsPathRoot = new File("${tempFolder.absolutePath}/${exportedJobZipNameWithoutExt}")
def jobsConfigFromExportedZip = SaagieClientUtils.extractJobConfigAndPackageFromExportedJob(exportedArtifactsPathRoot)
def pipelinesConfigFromExportedZip = SaagieClientUtils.extractPipelineConfigAndPackageFromExportedPipeline(exportedArtifactsPathRoot)
def variablesConfigFromExportedZip = SaagieClientUtils.extractVariableConfigAndPackageFromExportedVariable(exportedArtifactsPathRoot)
def response = [
status : 'success',
job : [],
pipeline : []
pipeline : [],
variable : []
]
def listJobs = null
def callbackJobToDebug = { newMappedJobData, job, id, versions = null ->
def processJobImportation = { newMappedJobData, job, id, versions = null ->
def jobToImport = new Job()
def jobVersionToImport = new JobVersion()
jobToImport = newMappedJobData.job
Expand Down Expand Up @@ -1549,7 +1683,7 @@ class SaagieClient {
}

def listPipelines = null
def callbackPipelinesToDebug = { newMappedPipeline, pipeline, id, versions, newlistJobs ->
def processPipelineToImport = { newMappedPipeline, pipeline, id, versions, newlistJobs ->
listPipelines = getPipelineListByNameAndId()
def pipelineToImport = newMappedPipeline.pipeline
def pipelineVersionToImport = newMappedPipeline.pipelineVersion
Expand Down Expand Up @@ -1609,23 +1743,55 @@ class SaagieClient {

}

def processVariableToImport = { newMappedVariable, variable, id ->

def newlyCreatedVariable = saveEnvironmentVariable(newMappedVariable)

response.variable << [
id : newlyCreatedVariable.id,
name : newlyCreatedVariable.name
]

}

if (jobsConfigFromExportedZip?.jobs) {
ImportJobService.importAndCreateJobs(
jobsConfigFromExportedZip.jobs,
configuration,
callbackJobToDebug
processJobImportation
)
}

if ( pipelinesConfigFromExportedZip?.pipelines && response.status == 'success') {
if (pipelinesConfigFromExportedZip?.pipelines && response.status == ResponseStatusEnum.success.name) {
def newlistJobs = getJobListByNameAndId()
ImportPipelineService.importAndCreatePipelines(
pipelinesConfigFromExportedZip.pipelines,
configuration,
callbackPipelinesToDebug,
processPipelineToImport,
newlistJobs
)
}

if (variablesConfigFromExportedZip?.variables && response.status == ResponseStatusEnum.success.name) {


variablesConfigFromExportedZip?.variables?.values()?.forEach { variable ->
def listVariablesByNameAndId = VariableService.instance.getVariableList(variable.configOverride, configuration.project.id, this.&getVariableEnvironmentFromList)
def foundVariable = listVariablesByNameAndId.find {
variable.name.equals(it.name) && variable.scope.equals(it.scope)
}

if (foundVariable) {
throw new GradleException("Environmnent varaible name : ${newMappedVariable.name} already existe in the targeted platform")
}
}

ImportVariableService.importAndCreateVariables(
variablesConfigFromExportedZip.variables,
processVariableToImport
)
}

return response
} catch (InvalidUserDataException invalidUserDataException) {
throw invalidUserDataException
Expand All @@ -1639,6 +1805,30 @@ class SaagieClient {
}
}

private getVariableEnvironmentFromList( newMappedVariable ) {
Request requestlistVariablesByNameAndId = null
def listVariables = null
def isProjectRequest = newMappedVariable.scope.equals(EnvVarScopeTypeEnum.project.name().toUpperCase())
if (isProjectRequest) {
requestlistVariablesByNameAndId = saagieUtils.getProjectEnvironmentVariables(configuration.project.id)
} else {
requestlistVariablesByNameAndId = saagieUtils.getGlobalVariableByNameAndId()
}
tryCatchClosure({
client.newCall(requestlistVariablesByNameAndId).execute().withCloseable { responseVariableList ->
handleErrors(responseVariableList)
String responseBodyForVariableList = responseVariableList.body().string()
logger.debug("variable list : $responseBodyForVariableList")
def parsedResultForVariableList = slurper.parseText(responseBodyForVariableList)
if (isProjectRequest) {
listVariables = parsedResultForVariableList.data.projectEnvironmentVariables
} else if (parsedResultForVariableList.data?.projectEnvironmentVariables) {
listVariables = parsedResultForVariableList.data.globalEnvironmentVariables
}
return listVariables
}
}, 'Unknown error in importArtifact Request', 'getVariableEnvironmentFromList Request')
}

private getJobListByNameAndId() {
def listJobs = null
Expand Down
26 changes: 26 additions & 0 deletions src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClientUtils.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ class SaagieClientUtils {
static final EXPORTED_JOB_CONFIG_FILENAME = 'job.json'
static final EXPORTED_JOB_PACKAGE_FOLDER_NAME = 'package'
static final EXPORTED_PIPELINE_CONFIG_FILENAME = 'pipeline.json'
static final EXPORTED_VARIABLE_CONFIG_FILENAME = 'variable.json'

/**
* run through all files in the provided folder, and return
Expand Down Expand Up @@ -100,4 +101,29 @@ class SaagieClientUtils {
}
return null
}

def static extractVariableConfigAndPackageFromExportedVariable( File exportedZipFolder ) {
Map extractedConfig = [
variables : [ : ],
]

File variablesFolder = new File( "${ exportedZipFolder.absolutePath }/Variable" )
if ( variablesFolder.exists() ) {
variablesFolder.eachFile { variableFolder ->
String variableId = variableFolder.name
String variableFolderPath = variableFolder.absolutePath

extractedConfig.variables[ variableId ] = [
configOverride : null
]

def jsonParser = new JsonSlurper()
File variableConfigFile = new File( "${ variableFolderPath }/${ EXPORTED_VARIABLE_CONFIG_FILENAME }" )
extractedConfig.variables[ variableId ].configOverride = jsonParser.parse( variableConfigFile )
}

return extractedConfig
}
return null
}
}
Loading

0 comments on commit 1caab16

Please sign in to comment.