From ce46066ba748fd9556f3f3bcd8f3e2f5ef4a6f2c Mon Sep 17 00:00:00 2001 From: duftler Date: Wed, 15 Jun 2016 15:22:50 -0400 Subject: [PATCH] Refactor rosco to run jobs directly instead of relying on rush. Drop dependency on cassandra. --- README.md | 2 +- build.gradle | 2 +- rosco-core/rosco-core.gradle | 1 + .../netflix/spinnaker/rosco/api/Bake.groovy | 2 + .../spinnaker/rosco/api/BakeStatus.groovy | 15 +- .../rosco/config/RoscoConfiguration.groovy | 52 ---- .../rosco/executor/BakePoller.groovy | 143 +++++---- .../JobExecutor.groovy} | 18 +- .../JobRequest.groovy} | 10 +- .../rosco/jobs/config/LocalJobConfig.groovy | 35 +++ .../rosco/jobs/local/JobExecutorLocal.groovy | 174 +++++++++++ .../rosco/persistence/BakeStore.groovy | 30 +- .../persistence/RedisBackedBakeStore.groovy | 145 +++++++-- .../providers/CloudProviderBakeHandler.groovy | 6 +- .../DockerFriendlyPackerCommandFactory.groovy | 39 --- .../util/PackerCommandFactory.groovy | 2 +- .../retrofit/RetrofitConfiguration.groovy | 52 ---- .../gson/GsonOptionalDeserializer.groovy | 38 --- .../rosco/rush/api/RushService.groovy | 39 --- .../rosco/rush/api/ScriptExecution.groovy | 42 --- .../rush/config/RushConfiguration.groovy | 77 ----- .../rush/health/RushHealthIndicator.groovy | 76 ----- .../rosco/executor/BakePollerSpec.groovy | 134 +++----- ...kerFriendlyPackerCommandFactorySpec.groovy | 124 -------- rosco-web/config/packer/install_packages.sh | 2 +- rosco-web/config/rosco.yml | 28 +- .../com/netflix/spinnaker/rosco/Main.groovy | 5 +- .../rosco/controllers/BakeryController.groovy | 91 +++--- .../controllers/BakeryControllerSpec.groovy | 285 +++++++++--------- 29 files changed, 706 insertions(+), 963 deletions(-) rename rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/{rush/api/ScriptId.groovy => jobs/JobExecutor.groovy} (62%) rename rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/{rush/api/ScriptRequest.groovy => jobs/JobRequest.groovy} (82%) create mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/config/LocalJobConfig.groovy create mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/local/JobExecutorLocal.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactory.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/RetrofitConfiguration.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/gson/GsonOptionalDeserializer.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/RushService.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptExecution.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/config/RushConfiguration.groovy delete mode 100644 rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/health/RushHealthIndicator.groovy delete mode 100644 rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactorySpec.groovy diff --git a/README.md b/README.md index 7a9c6eee8..de04e2e4c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ git clone git@github.com:spinnaker/spinnaker.git docker-machine create --virtualbox-disk-size 8192 --virtualbox-memory 8192 -d virtualbox spinnaker eval $(docker-machine env spinnaker) cd spinnaker/experimental/docker-compose -docker-compose up -d redis rush +docker-compose up -d redis ``` ## Verify redis diff --git a/build.gradle b/build.gradle index 48a22b6bf..e9232953e 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ allprojects { group = "com.netflix.spinnaker.rosco" spinnaker { - dependenciesVersion = "0.19.0" + dependenciesVersion = "0.40.0" } configurations.all { diff --git a/rosco-core/rosco-core.gradle b/rosco-core/rosco-core.gradle index 79793acbe..128c82344 100644 --- a/rosco-core/rosco-core.gradle +++ b/rosco-core/rosco-core.gradle @@ -3,6 +3,7 @@ dependencies { spinnaker.group('retrofitDefault') compile spinnaker.dependency('rxJava') spinnaker.group('jackson') + compile spinnaker.dependency("commonsExec") compile spinnaker.dependency("frigga") compile spinnaker.dependency('jacksonGuava') compile spinnaker.dependency('jedis') diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/Bake.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/Bake.groovy index 05bee1140..7a6886f13 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/Bake.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/Bake.groovy @@ -19,6 +19,7 @@ package com.netflix.spinnaker.rosco.api import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import io.swagger.annotations.ApiModelProperty /** * The details of a completed bake. @@ -29,6 +30,7 @@ import groovy.transform.ToString @EqualsAndHashCode(includes = "id") @ToString(includeNames = true) class Bake { + @ApiModelProperty(value="The id of the bake job.") String id String ami String image_name diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/BakeStatus.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/BakeStatus.groovy index 11081b5e0..ca9115a25 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/BakeStatus.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/BakeStatus.groovy @@ -16,9 +16,11 @@ package com.netflix.spinnaker.rosco.api +import com.fasterxml.jackson.annotation.JsonIgnore import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import io.swagger.annotations.ApiModelProperty /** * The state of a bake as returned by the Bakery API when a bake is created. Once complete it provides a link to the @@ -32,6 +34,7 @@ class BakeStatus implements Serializable { /** * The bake status id. */ + @ApiModelProperty(value="The id of the bake request.") String id State state @@ -43,10 +46,20 @@ class BakeStatus implements Serializable { * * @see BakeryController#lookupBake */ + @ApiModelProperty(value="The id of the bake job. Can be passed to lookupBake() to retrieve the details of the newly-baked image.") String resource_id + @JsonIgnore + String logsContent + + @JsonIgnore + long createdTimestamp + + @JsonIgnore + long updatedTimestamp + static enum State { - PENDING, RUNNING, COMPLETED, SUSPENDED, CANCELED + RUNNING, COMPLETED, CANCELED } static enum Result { diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy index 74064e11f..cee99dbf3 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy @@ -17,22 +17,18 @@ package com.netflix.spinnaker.rosco.config import com.fasterxml.jackson.databind.ObjectMapper -import com.netflix.spinnaker.rosco.api.BakeStatus import com.netflix.spinnaker.rosco.executor.BakePoller import com.netflix.spinnaker.rosco.persistence.BakeStore import com.netflix.spinnaker.rosco.persistence.RedisBackedBakeStore import com.netflix.spinnaker.rosco.providers.registry.CloudProviderBakeHandlerRegistry import com.netflix.spinnaker.rosco.providers.registry.DefaultCloudProviderBakeHandlerRegistry import com.netflix.spinnaker.rosco.providers.util.DefaultImageNameFactory -import com.netflix.spinnaker.rosco.providers.util.DockerFriendlyPackerCommandFactory import com.netflix.spinnaker.rosco.providers.util.ImageNameFactory import com.netflix.spinnaker.rosco.providers.util.LocalJobFriendlyPackerCommandFactory import com.netflix.spinnaker.rosco.providers.util.PackerCommandFactory import groovy.transform.CompileStatic import org.springframework.beans.factory.config.ConfigurableBeanFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Scope @@ -89,58 +85,10 @@ class RoscoConfiguration { return new LocalJobFriendlyPackerCommandFactory() } - @Bean - @ConditionalOnProperty('rush.docker.enabled') - PackerCommandFactory dockerFriendlyPackerCommandFactory() { - return new DockerFriendlyPackerCommandFactory() - } - @Bean @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) ObjectMapper mapper() { return new ObjectMapper() } - @Bean - @ConfigurationProperties('executionStatusToBakeStates') - ExecutionStatusToBakeStateMap executionStatusToBakeStateMap() { - new ExecutionStatusToBakeStateMap() - } - - static class ExecutionStatusToBakeStateMap { - List associations - - public BakeStatus.State convertExecutionStatusToBakeState(String executionStatus) { - associations.find { - it.executionStatus == executionStatus - }?.bakeState - } - } - - static class ExecutionStatusToBakeState { - String executionStatus - BakeStatus.State bakeState - } - - @Bean - @ConfigurationProperties('executionStatusToBakeResults') - ExecutionStatusToBakeResultMap executionStatusToBakeResultMap() { - new ExecutionStatusToBakeResultMap() - } - - static class ExecutionStatusToBakeResultMap { - List associations - - public BakeStatus.Result convertExecutionStatusToBakeResult(String executionStatus) { - associations.find { - it.executionStatus == executionStatus - }?.bakeResult - } - } - - static class ExecutionStatusToBakeResult { - String executionStatus - BakeStatus.Result bakeResult - } - } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/executor/BakePoller.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/executor/BakePoller.groovy index c08c50ff9..b63eca39c 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/executor/BakePoller.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/executor/BakePoller.groovy @@ -18,20 +18,15 @@ package com.netflix.spinnaker.rosco.executor import com.netflix.spinnaker.rosco.api.Bake import com.netflix.spinnaker.rosco.api.BakeStatus -import com.netflix.spinnaker.rosco.config.RoscoConfiguration +import com.netflix.spinnaker.rosco.jobs.JobExecutor import com.netflix.spinnaker.rosco.persistence.BakeStore import com.netflix.spinnaker.rosco.providers.registry.CloudProviderBakeHandlerRegistry -import com.netflix.spinnaker.rosco.rush.api.RushService -import com.netflix.spinnaker.rosco.rush.api.ScriptExecution -import com.netflix.spinnaker.rosco.rush.api.ScriptRequest import groovy.util.logging.Slf4j import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.context.ApplicationListener import org.springframework.context.event.ContextRefreshedEvent import org.springframework.stereotype.Component -import retrofit.RetrofitError -import retrofit.mime.TypedByteArray import rx.functions.Action0 import rx.schedulers.Schedulers @@ -39,78 +34,127 @@ import java.util.concurrent.TimeUnit /** * BakePoller periodically queries the bake store for incomplete bakes. For each incomplete bake, it queries - * the scripting engine for an up-to-date status and logs. The status and logs are then persisted via the bake + * the job executor for an up-to-date status and logs. The status and logs are then persisted via the bake * store. When a bake completes, it is the BakePoller that persists the completed bake details via the bake store. - * The polling interval defaults to 15 seconds and can be overridden by specifying the pollingIntervalSeconds - * property. + * The polling interval defaults to 15 seconds and can be overridden by specifying the + * rosco.polling.pollingIntervalSeconds property. */ @Slf4j @Component class BakePoller implements ApplicationListener { - @Value('${pollingIntervalSeconds:15}') - int pollingIntervalSeconds - @Autowired - BakeStore bakeStore + String roscoInstanceId - @Autowired - ScriptRequest baseScriptRequest + @Value('${rosco.polling.pollingIntervalSeconds:15}') + int pollingIntervalSeconds - @Autowired - RushService rushService + @Value('${rosco.polling.orphanedJobPollingIntervalSeconds:30}') + int orphanedJobPollingIntervalSeconds + + @Value('${rosco.polling.orphanedJobTimeoutMinutes:30}') + long orphanedJobTimeoutMinutes @Autowired - CloudProviderBakeHandlerRegistry cloudProviderBakeHandlerRegistry + BakeStore bakeStore @Autowired - RoscoConfiguration.ExecutionStatusToBakeStateMap executionStatusToBakeStateMap + JobExecutor executor @Autowired - RoscoConfiguration.ExecutionStatusToBakeResultMap executionStatusToBakeResultMap + CloudProviderBakeHandlerRegistry cloudProviderBakeHandlerRegistry @Override void onApplicationEvent(ContextRefreshedEvent event) { + log.info("Starting polling agent for rosco instance $roscoInstanceId...") + + // Update this rosco instance's incomplete bakes. Schedulers.io().createWorker().schedulePeriodically( { - try { - rx.Observable.from(bakeStore.incompleteBakeIds) - .subscribe( - { String statusId -> - updateBakeStatusAndLogs(statusId) + rx.Observable.from(bakeStore.thisInstanceIncompleteBakeIds) + .subscribe( + { String incompleteBakeId -> + try { + updateBakeStatusAndLogs(incompleteBakeId) + } catch (Exception e) { + log.error("Polling Error:", e) + } }, { log.error("Error: ${it.message}") }, {} as Action0 - ) - } catch (Exception e) { - log.error("Polling Error:", e) - } + ) } as Action0, 0, pollingIntervalSeconds, TimeUnit.SECONDS ) + + // Check _all_ rosco instances' incomplete bakes for staleness. + Schedulers.io().createWorker().schedulePeriodically( + { + rx.Observable.from(bakeStore.allIncompleteBakeIds.entrySet()) + .subscribe( + { Map.Entry> entry -> + String roscoInstanceId = entry.key + Set incompleteBakeIds = entry.value + + if (roscoInstanceId != this.roscoInstanceId) { + try { + rx.Observable.from(incompleteBakeIds) + .subscribe( + { String statusId -> + BakeStatus bakeStatus = bakeStore.retrieveBakeStatusById(statusId) + + // The updatedTimestamp key will not be present if the in-flight bake is managed by an + // older-style (i.e. rosco/rush) rosco instance. + if (bakeStatus?.updatedTimestamp) { + long currentTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(bakeStore.timeInMilliseconds) + long lastUpdatedTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(bakeStatus.updatedTimestamp) + long eTimeMinutes = TimeUnit.SECONDS.toMinutes(currentTimeSeconds - lastUpdatedTimeSeconds) + + if (eTimeMinutes >= orphanedJobTimeoutMinutes) { + log.info("The staleness of bake $statusId ($eTimeMinutes minutes) has met or exceeded the " + + "value of orphanedJobTimeoutMinutes ($orphanedJobTimeoutMinutes minutes).") + + boolean cancellationSucceeded = bakeStore.cancelBakeById(statusId) + + if (!cancellationSucceeded) { + bakeStore.removeFromIncompletes(roscoInstanceId, statusId) + } + } + } + }, + { + log.error("Error: ${it.message}") + }, + {} as Action0 + ) + } catch (Exception e) { + log.error("Polling Error:", e) + } + } + }, + { + log.error("Error: ${it.message}") + }, + {} as Action0 + ) + } as Action0, 0, orphanedJobPollingIntervalSeconds, TimeUnit.SECONDS + ) } - // TODO(duftler): Support retries here, or at least some number of regular-interval communication failures before - // considering the bake a failure. void updateBakeStatusAndLogs(String statusId) { - try { - ScriptExecution scriptExecution = rushService.scriptDetails(statusId).toBlocking().single() - Map logsContentMap = rushService.getLogs(statusId, baseScriptRequest).toBlocking().single() - BakeStatus.State state = executionStatusToBakeStateMap.convertExecutionStatusToBakeState(scriptExecution.status) + BakeStatus bakeStatus = executor.updateJob(statusId) - if (state == BakeStatus.State.COMPLETED) { - completeBake(statusId, logsContentMap?.logsContent) + if (bakeStatus) { + if (bakeStatus.state == BakeStatus.State.COMPLETED) { + completeBake(statusId, bakeStatus.logsContent) } - bakeStore.updateBakeStatus(new BakeStatus(id: scriptExecution.id, - resource_id: scriptExecution.id, - state: state, - result: executionStatusToBakeResultMap.convertExecutionStatusToBakeResult(scriptExecution.status)), - logsContentMap) - } catch (RetrofitError e) { - handleRetrofitError(e, "Unable to retrieve status for '$statusId'.", statusId) - + bakeStore.updateBakeStatus(bakeStatus) + } else { + String errorMessage = "Unable to retrieve status for '$statusId'." + log.error(errorMessage) + bakeStore.storeBakeError(statusId, errorMessage) bakeStore.cancelBakeById(statusId) } } @@ -139,13 +183,4 @@ class BakePoller implements ApplicationListener { log.error("Unable to retrieve bake details for '$bakeId'.") } - - private handleRetrofitError(RetrofitError e, String errMessage, String id) { - log.error(errMessage, e) - - def errorBytes = ((TypedByteArray)e?.response?.body)?.bytes - def errorMessage = errorBytes ? new String(errorBytes) : "{}" - - bakeStore.storeBakeError(id, errorMessage) - } } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptId.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobExecutor.groovy similarity index 62% rename from rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptId.groovy rename to rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobExecutor.groovy index b7f07fd79..7f07f75d8 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptId.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobExecutor.groovy @@ -1,11 +1,11 @@ /* - * Copyright 2015 Google, Inc. + * Copyright 2016 Google, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,10 +14,12 @@ * limitations under the License. */ -package com.netflix.spinnaker.rosco.rush.api +package com.netflix.spinnaker.rosco.jobs -class ScriptId { +import com.netflix.spinnaker.rosco.api.BakeStatus - String id - -} +interface JobExecutor { + String startJob(JobRequest jobRequest) + BakeStatus updateJob(String jobId) + void cancelJob(String jobId) +} \ No newline at end of file diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptRequest.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobRequest.groovy similarity index 82% rename from rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptRequest.groovy rename to rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobRequest.groovy index cb0ec874e..e31a0eb86 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptRequest.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobRequest.groovy @@ -14,23 +14,17 @@ * limitations under the License. */ -package com.netflix.spinnaker.rosco.rush.api +package com.netflix.spinnaker.rosco.jobs import groovy.transform.CompileStatic import groovy.transform.Immutable /** * A request to bake a new machine image. - * - * @see RushService#runScript */ @Immutable(copyWith = true) @CompileStatic -class ScriptRequest { - String command +class JobRequest { List tokenizedCommand - String image - String credentials - boolean privileged } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/config/LocalJobConfig.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/config/LocalJobConfig.groovy new file mode 100644 index 000000000..acf0b506b --- /dev/null +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/config/LocalJobConfig.groovy @@ -0,0 +1,35 @@ +/* + * Copyright 2016 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.jobs.config + +import com.netflix.spinnaker.rosco.jobs.JobExecutor +import com.netflix.spinnaker.rosco.jobs.local.JobExecutorLocal +import groovy.util.logging.Slf4j +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Slf4j +@Configuration +class LocalJobConfig { + + @Bean + @ConditionalOnMissingBean(JobExecutor) + JobExecutor jobExecutorLocal() { + new JobExecutorLocal() + } +} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/local/JobExecutorLocal.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/local/JobExecutorLocal.groovy new file mode 100644 index 000000000..5758caa40 --- /dev/null +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/local/JobExecutorLocal.groovy @@ -0,0 +1,174 @@ +/* + * Copyright 2016 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.jobs.local + +import com.netflix.spinnaker.rosco.api.BakeStatus +import com.netflix.spinnaker.rosco.jobs.JobExecutor +import com.netflix.spinnaker.rosco.jobs.JobRequest +import groovy.util.logging.Slf4j +import org.apache.commons.exec.* +import org.springframework.beans.factory.annotation.Value +import rx.Scheduler +import rx.functions.Action0 +import rx.schedulers.Schedulers + +import java.util.concurrent.ConcurrentHashMap + +@Slf4j +class JobExecutorLocal implements JobExecutor { + + @Value('${rosco.jobs.local.timeoutMinutes:10}') + long timeoutMinutes + + Scheduler scheduler = Schedulers.computation() + Map jobIdToHandlerMap = new ConcurrentHashMap() + + @Override + String startJob(JobRequest jobRequest) { + log.info("Starting job: $jobRequest.tokenizedCommand...") + + String jobId = UUID.randomUUID().toString() + + scheduler.createWorker().schedule( + new Action0() { + @Override + public void call() { + ByteArrayOutputStream stdOutAndErr = new ByteArrayOutputStream() + PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(stdOutAndErr) + CommandLine commandLine + + if (jobRequest.tokenizedCommand) { + log.info("Executing $jobId with tokenized command: $jobRequest.tokenizedCommand") + + // Grab the first element as the command. + commandLine = new CommandLine(jobRequest.tokenizedCommand[0]) + + // Treat the rest as arguments. + String[] arguments = Arrays.copyOfRange(jobRequest.tokenizedCommand.toArray(), 1, jobRequest.tokenizedCommand.size()) + + commandLine.addArguments(arguments, false) + } else { + log.info("No tokenizedCommand specified for $jobId.") + + throw new IllegalArgumentException("No tokenizedCommand specified for $jobId.") + } + + DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler() + ExecuteWatchdog watchdog = new ExecuteWatchdog(timeoutMinutes * 60 * 1000){ + @Override + void timeoutOccured(Watchdog w) { + // If a watchdog is passed in, this was an actual time-out. Otherwise, it is likely + // the result of calling watchdog.destroyProcess(). + if (w) { + log.info("Job $jobId timed-out (after $timeoutMinutes minutes).") + + cancelJob(jobId) + } + + super.timeoutOccured(w) + } + } + Executor executor = new DefaultExecutor() + executor.setStreamHandler(pumpStreamHandler) + executor.setWatchdog(watchdog) + executor.execute(commandLine, resultHandler) + + // Give the job some time to spin up. + sleep(500) + + jobIdToHandlerMap.put(jobId, [ + handler: resultHandler, + watchdog: watchdog, + stdOutAndErr: stdOutAndErr + ]) + } + } + ) + + return jobId + } + + @Override + BakeStatus updateJob(String jobId) { + try { + log.info("Polling state for $jobId...") + + if (jobIdToHandlerMap[jobId]) { + BakeStatus bakeStatus = new BakeStatus(id: jobId, resource_id: jobId) + + DefaultExecuteResultHandler resultHandler + ByteArrayOutputStream stdOutAndErr + + jobIdToHandlerMap[jobId].with { + resultHandler = it.handler + stdOutAndErr = it.stdOutAndErr + } + + String logsContent = new String(stdOutAndErr.toByteArray()) + + if (resultHandler.hasResult()) { + log.info("State for $jobId changed with exit code $resultHandler.exitValue.") + + if (!logsContent) { + logsContent = resultHandler.exception ? resultHandler.exception.message : "No output from command." + } + + if (resultHandler.exitValue == 0) { + bakeStatus.state = BakeStatus.State.COMPLETED + bakeStatus.result = BakeStatus.Result.SUCCESS + } else { + bakeStatus.state = BakeStatus.State.CANCELED + bakeStatus.result = BakeStatus.Result.FAILURE + } + + jobIdToHandlerMap.remove(jobId) + } else { + bakeStatus.state = BakeStatus.State.RUNNING + } + + if (logsContent) { + bakeStatus.logsContent = logsContent + } + + return bakeStatus + } else { + // This instance of rosco is not managing the job. + return null + } + } catch (Exception e) { + log.error("Failed to update $jobId", e) + + return null + } + + } + + @Override + void cancelJob(String jobId) { + log.info("Canceling job $jobId...") + + // Terminate the process. + if (jobIdToHandlerMap[jobId]) { + jobIdToHandlerMap[jobId].watchdog.destroyProcess() + } + + // Remove the job from this rosco instance's handler map. + jobIdToHandlerMap.remove(jobId) + + // The next polling interval will be unable to retrieve the job status and will mark the bake as canceled. + } +} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/BakeStore.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/BakeStore.groovy index aaa5a3295..5a84e82d3 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/BakeStore.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/BakeStore.groovy @@ -34,7 +34,7 @@ interface BakeStore { * Store the region, bakeRequest and bakeStatus in association with both the bakeKey and bakeId. If bake key * has already been set, return a bakeStatus with that bake's id instead. None of the arguments may be null. */ - public BakeStatus storeNewBakeStatus(String bakeKey, String region, BakeRequest bakeRequest, String bakeId) + public BakeStatus storeNewBakeStatus(String bakeKey, String region, BakeRequest bakeRequest, BakeStatus bakeStatus, String command) /** * Update the completed bake details associated with both the bakeKey and bakeDetails.id. bakeDetails may not be null. @@ -47,12 +47,6 @@ interface BakeStore { */ public void updateBakeStatus(BakeStatus bakeStatus) - /** - * Update the bakeStatus and bakeLogs associated with both the bakeKey and bakeStatus.id. If bakeStatus.state is - * neither pending nor running, remove bakeStatus.id from the set of incomplete bakes. bakeStatus may not be null. - */ - public void updateBakeStatus(BakeStatus bakeStatus, Map logsContent) - /** * Store the error in association with both the bakeId and the bakeKey. Neither argument may be null. */ @@ -90,14 +84,28 @@ interface BakeStore { public boolean deleteBakeByKey(String bakeKey) /** - * Cancel the incomplete bake associated with the bake id. Delete the bake status, completed bake details and logs - * associated with the bake id. If the bake is still incomplete, remove the bake id from the set of incomplete bakes. + * Cancel the incomplete bake associated with the bake id and delete the completed bake details associated with the + * bake id. If the bake is still incomplete, remove the bake id from the set of incomplete bakes. */ public boolean cancelBakeById(String bakeId) /** - * Retrieve the set of incomplete bake ids. + * Remove the incomplete bake from the rosco instance's set of incomplete bakes. + */ + public void removeFromIncompletes(String roscoInstanceId, String bakeId) + + /** + * Retrieve the set of incomplete bake ids for this rosco instance. */ - public Set getIncompleteBakeIds() + public Set getThisInstanceIncompleteBakeIds() + /** + * Retrieve a map of rosco instance ids -> sets of incomplete bake ids. + */ + public Map> getAllIncompleteBakeIds() + + /** + * Get the current redis server time in milliseconds. + */ + public long getTimeInMilliseconds() } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/RedisBackedBakeStore.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/RedisBackedBakeStore.groovy index 1415664ea..b9fd362b7 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/RedisBackedBakeStore.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/RedisBackedBakeStore.groovy @@ -25,8 +25,12 @@ import org.springframework.beans.factory.annotation.Autowired import redis.clients.jedis.JedisPool import redis.clients.jedis.exceptions.JedisDataException +import java.util.concurrent.TimeUnit + class RedisBackedBakeStore implements BakeStore { + public static final String INCOMPLETE_BAKES_PREFIX = "allBakes:incomplete:" + @Autowired String roscoInstanceId @@ -70,7 +74,6 @@ class RedisBackedBakeStore implements BakeStore { redis.call('ZADD', KEYS[1], ARGV[1], KEYS[3]) -- If we lost a race to initiate a new bake, just return the race winner's bake status. - -- TODO(duftler): When rush supports cancelling an in-flight script execution, employ that here. if redis.call('EXISTS', KEYS[3]) == 1 then return redis.call('HMGET', KEYS[3], 'bakeStatus') end @@ -81,7 +84,11 @@ class RedisBackedBakeStore implements BakeStore { 'region', ARGV[2], 'bakeRequest', ARGV[3], 'bakeStatus', ARGV[4], - 'creationTimestamp', ARGV[1]) + 'bakeLogs', ARGV[5], + 'command', ARGV[6], + 'roscoInstanceId', ARGV[7], + 'createdTimestamp', ARGV[1], + 'updatedTimestamp', ARGV[1]) -- Set bake key hash values. redis.call('HMSET', KEYS[3], @@ -89,7 +96,11 @@ class RedisBackedBakeStore implements BakeStore { 'region', ARGV[2], 'bakeRequest', ARGV[3], 'bakeStatus', ARGV[4], - 'creationTimestamp', ARGV[1]) + 'bakeLogs', ARGV[5], + 'command', ARGV[6], + 'roscoInstanceId', ARGV[7], + 'createdTimestamp', ARGV[1], + 'updatedTimestamp', ARGV[1]) -- Add bake id to set of incomplete bakes. redis.call('SADD', KEYS[4], KEYS[2]) @@ -98,6 +109,13 @@ class RedisBackedBakeStore implements BakeStore { redis.call('DEL', KEYS[5]) """) updateBakeDetailsSHA = jedis.scriptLoad("""\ + local existing_bake_status = redis.call('HGET', KEYS[1], 'bakeStatus') + + -- Ensure we don't update/resurrect a canceled bake (can happen due to a race). + if existing_bake_status and cjson.decode(existing_bake_status)["state"] == "CANCELED" then + return + end + -- Retrieve the bake key associated with bake id. local bake_key = redis.call('HGET', KEYS[1], 'bakeKey') @@ -108,19 +126,28 @@ class RedisBackedBakeStore implements BakeStore { redis.call('HSET', bake_key, 'bakeDetails', ARGV[1]) """) def updateBakeStatusBaseScript = """\ + local existing_bake_status = redis.call('HGET', KEYS[1], 'bakeStatus') + + -- Ensure we don't update/resurrect a canceled bake (can happen due to a race). + if existing_bake_status and cjson.decode(existing_bake_status)["state"] == "CANCELED" then + return + end + -- Retrieve the bake key associated with bake id. local bake_key = redis.call('HGET', KEYS[1], 'bakeKey') -- Update the bake status and logs set on the bake id hash. redis.call('HMSET', KEYS[1], 'bakeStatus', ARGV[1], - 'bakeLogs', ARGV[2]) + 'bakeLogs', ARGV[2], + 'updatedTimestamp', ARGV[3]) if bake_key then -- Update the bake status and logs set on the bake key hash. redis.call('HMSET', bake_key, 'bakeStatus', ARGV[1], - 'bakeLogs', ARGV[2]) + 'bakeLogs', ARGV[2], + 'updatedTimestamp', ARGV[3]) end """ updateBakeStatusSHA = jedis.scriptLoad(updateBakeStatusBaseScript) @@ -165,15 +192,22 @@ class RedisBackedBakeStore implements BakeStore { -- Retrieve the bake key associated with bake id. local bake_key = redis.call('HGET', KEYS[1], 'bakeKey') - -- Remove bake id from the set of incomplete bakes. - local ret = redis.call('SREM', KEYS[2], KEYS[1]) + -- Retrieve the rosco instance id associated with bake id. + local rosco_instance_id = redis.call('HGET', KEYS[1], 'roscoInstanceId') + + local ret + + if rosco_instance_id then + -- Remove bake id from that rosco instance's set of incomplete bakes. + ret = redis.call('SREM', 'allBakes:incomplete:' .. rosco_instance_id, KEYS[1]) + end -- Update the bake status set on the bake id hash. redis.call('HSET', KEYS[1], 'bakeStatus', ARGV[1]) if bake_key then -- Remove the bake key from the set of bakes. - redis.call('ZREM', KEYS[3], bake_key) + redis.call('ZREM', KEYS[2], bake_key) -- Delete the bake key key. redis.call('DEL', bake_key) @@ -185,8 +219,12 @@ class RedisBackedBakeStore implements BakeStore { } } - private String getIncompleteBakesKey() { - return "allBakes:incomplete:$roscoInstanceId".toString() + private String getAllIncompleteBakesKeyPattern() { + return "$INCOMPLETE_BAKES_PREFIX*" + } + + private String getThisInstanceIncompleteBakesKey() { + return "$INCOMPLETE_BAKES_PREFIX$roscoInstanceId".toString() } @Override @@ -200,14 +238,14 @@ class RedisBackedBakeStore implements BakeStore { } @Override - public BakeStatus storeNewBakeStatus(String bakeKey, String region, BakeRequest bakeRequest, String bakeId) { + public BakeStatus storeNewBakeStatus(String bakeKey, String region, BakeRequest bakeRequest, BakeStatus bakeStatus, String command) { def lockKey = "lock:$bakeKey" def bakeRequestJson = mapper.writeValueAsString(bakeRequest) - def bakeStatus = new BakeStatus(id: bakeId, resource_id: bakeId, state: BakeStatus.State.PENDING) def bakeStatusJson = mapper.writeValueAsString(bakeStatus) - def creationTimestamp = System.currentTimeMillis() - def keyList = ["allBakes", bakeId, bakeKey, incompleteBakesKey, lockKey.toString()] - def argList = [creationTimestamp.toString(), region, bakeRequestJson, bakeStatusJson] + def bakeLogsJson = mapper.writeValueAsString(bakeStatus.logsContent ? [logsContent: bakeStatus.logsContent] : [:]) + def createdTimestampMilliseconds = timeInMilliseconds + def keyList = ["allBakes", bakeStatus.id, bakeKey, thisInstanceIncompleteBakesKey, lockKey.toString()] + def argList = [createdTimestampMilliseconds + "", region, bakeRequestJson, bakeStatusJson, bakeLogsJson, command, roscoInstanceId] def result = evalSHA("storeNewBakeStatusSHA", keyList, argList) // Check if the script returned a bake status set by the winner of a race. @@ -228,17 +266,18 @@ class RedisBackedBakeStore implements BakeStore { } @Override - public void updateBakeStatus(BakeStatus bakeStatus, Map logsContent=[:]) { + public void updateBakeStatus(BakeStatus bakeStatus) { def bakeStatusJson = mapper.writeValueAsString(bakeStatus) - def bakeLogsJson = mapper.writeValueAsString(logsContent ?: [:]) + def bakeLogsJson = mapper.writeValueAsString(bakeStatus.logsContent ? [logsContent: bakeStatus.logsContent] : [:]) + def updatedTimestampMilliseconds = timeInMilliseconds def scriptSHA = "updateBakeStatusSHA" - if (bakeStatus.state != BakeStatus.State.PENDING && bakeStatus.state != BakeStatus.State.RUNNING) { + if (bakeStatus.state != BakeStatus.State.RUNNING) { scriptSHA = "updateBakeStatusWithIncompleteRemovalSHA" } - def keyList = [bakeStatus.id, incompleteBakesKey] - def argList = [bakeStatusJson, bakeLogsJson] + def keyList = [bakeStatus.id, thisInstanceIncompleteBakesKey] + def argList = [bakeStatusJson, bakeLogsJson, updatedTimestampMilliseconds + ""] evalSHA(scriptSHA, keyList, argList) } @@ -276,9 +315,18 @@ class RedisBackedBakeStore implements BakeStore { def jedis = jedisPool.getResource() jedis.withCloseable { - def bakeStatusJson = jedis.hget(bakeId, "bakeStatus") + def (String bakeStatusJson, + String createdTimestampStr, + String updatedTimestampStr) = jedis.hmget(bakeId, "bakeStatus", "createdTimestamp", "updatedTimestamp") - return bakeStatusJson ? mapper.readValue(bakeStatusJson, BakeStatus) : null + BakeStatus bakeStatus = bakeStatusJson ? mapper.readValue(bakeStatusJson, BakeStatus) : null + + if (bakeStatus && createdTimestampStr) { + bakeStatus.createdTimestamp = Long.parseLong(createdTimestampStr) + bakeStatus.updatedTimestamp = Long.parseLong(updatedTimestampStr) + } + + return bakeStatus } } @@ -306,7 +354,7 @@ class RedisBackedBakeStore implements BakeStore { @Override public boolean deleteBakeByKey(String bakeKey) { - def keyList = [bakeKey, "allBakes", incompleteBakesKey] + def keyList = [bakeKey, "allBakes", thisInstanceIncompleteBakesKey] return evalSHA("deleteBakeByKeySHA", keyList, []) == 1 } @@ -317,19 +365,64 @@ class RedisBackedBakeStore implements BakeStore { resource_id: bakeId, state: BakeStatus.State.CANCELED, result: BakeStatus.Result.FAILURE) + def jedis = jedisPool.getResource() def bakeStatusJson = mapper.writeValueAsString(bakeStatus) - def keyList = [bakeId, incompleteBakesKey, "allBakes"] + def keyList = [bakeId, "allBakes"] + + jedis.withCloseable { + Set incompleteBakesKeys = jedis.keys(allIncompleteBakesKeyPattern) + + keyList += incompleteBakesKeys + } + def argList = [bakeStatusJson] return evalSHA("cancelBakeByIdSHA", keyList, argList) == 1 } @Override - public Set getIncompleteBakeIds() { + public void removeFromIncompletes(String roscoInstanceId, String bakeId) { + def jedis = jedisPool.getResource() + + jedis.withCloseable { + jedis.srem("$INCOMPLETE_BAKES_PREFIX$roscoInstanceId", bakeId) + } + } + + @Override + public Set getThisInstanceIncompleteBakeIds() { + def jedis = jedisPool.getResource() + + jedis.withCloseable { + return jedis.smembers(thisInstanceIncompleteBakesKey) + } + } + + @Override + public Map> getAllIncompleteBakeIds() { + def jedis = jedisPool.getResource() + + jedis.withCloseable { + Set incompleteBakesKeys = jedis.keys(allIncompleteBakesKeyPattern) + + return incompleteBakesKeys.collectEntries { incompleteBakesKey -> + String roscoInstanceId = incompleteBakesKey.substring(INCOMPLETE_BAKES_PREFIX.length()) + + [(roscoInstanceId): jedis.smembers(incompleteBakesKey)] + } + } + } + + @Override + public long getTimeInMilliseconds() { def jedis = jedisPool.getResource() jedis.withCloseable { - return jedis.smembers(incompleteBakesKey) + def (String timeSecondsStr, String microsecondsStr) = jedis.time() + long timeSeconds = Long.parseLong(timeSecondsStr) + long microseconds = Long.parseLong(microsecondsStr) + + return TimeUnit.SECONDS.toMillis(timeSeconds) + TimeUnit.MICROSECONDS.toMillis(microseconds) } } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/CloudProviderBakeHandler.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/CloudProviderBakeHandler.groovy index 9a7653069..7ec12d4d1 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/CloudProviderBakeHandler.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/CloudProviderBakeHandler.groovy @@ -27,7 +27,7 @@ import org.springframework.beans.factory.annotation.Value abstract class CloudProviderBakeHandler { - @Value('${rush.configDir}') + @Value('${rosco.configDir}') String configDir @Autowired @@ -134,7 +134,7 @@ abstract class CloudProviderBakeHandler { abstract String getTemplateFileName() /** - * Build provider-specific script command for packer. + * Build provider-specific command for packer. */ List producePackerCommand(String region, BakeRequest bakeRequest) { def virtualizationSettings = findVirtualizationSettings(region, bakeRequest) @@ -151,8 +151,6 @@ abstract class CloudProviderBakeHandler { } parameterMap.package_type = selectedOptions.baseImage.packageType.packageType - - // TODO(duftler): Build out proper support for installation of packages. parameterMap.packages = packagesParameter if (appVersionStr) { diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactory.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactory.groovy deleted file mode 100644 index 8ab99d44d..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactory.groovy +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.providers.util - -class DockerFriendlyPackerCommandFactory implements PackerCommandFactory { - - @Override - List buildPackerCommand(String baseCommand, Map parameterMap, String templateFile) { - def packerCommand = ["sh", "-c"] - def shellCommandStr = baseCommand + "packer build -color=false" - - parameterMap.each { key, value -> - if (key && value) { - def keyValuePair = value instanceof String && value.contains(" ") ? "\"$key=$value\"" : "$key=$value" - - shellCommandStr += " -var $keyValuePair" - } - } - - shellCommandStr += " $templateFile" - packerCommand << shellCommandStr - packerCommand - } - -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/PackerCommandFactory.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/PackerCommandFactory.groovy index 009218253..fef64e403 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/PackerCommandFactory.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/util/PackerCommandFactory.groovy @@ -19,7 +19,7 @@ package com.netflix.spinnaker.rosco.providers.util interface PackerCommandFactory { /** - * Serialize passed parameters into a tokenized command string suitable for launching packer via rush. + * Serialize passed parameters into a tokenized command string suitable for launching packer. */ List buildPackerCommand(String baseCommand, Map parameterMap, String absoluteTemplateFilePath) diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/RetrofitConfiguration.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/RetrofitConfiguration.groovy deleted file mode 100644 index 634b8799a..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/RetrofitConfiguration.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.retrofit - -//import com.netflix.spinnaker.orca.retrofit.exceptions.RetrofitExceptionHandler -import groovy.transform.CompileStatic -import com.google.common.base.Optional -import com.google.gson.Gson -import com.google.gson.GsonBuilder -import com.netflix.spinnaker.rosco.retrofit.gson.GsonOptionalDeserializer -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import retrofit.RestAdapter.LogLevel -import retrofit.client.Client -import retrofit.client.OkClient - -@Configuration -@CompileStatic -class RetrofitConfiguration { - @Bean Client retrofitClient() { - new OkClient() - } - - @Bean LogLevel retrofitLogLevel() { - LogLevel.FULL - } - - @Bean Gson gson() { - new GsonBuilder() - .registerTypeAdapter(Optional, new GsonOptionalDeserializer()) - .create() - } - -// @Bean RetrofitExceptionHandler retrofitExceptionHandler() { -// new RetrofitExceptionHandler() -// } - -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/gson/GsonOptionalDeserializer.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/gson/GsonOptionalDeserializer.groovy deleted file mode 100644 index 89877df17..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/retrofit/gson/GsonOptionalDeserializer.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.retrofit.gson - -import java.lang.reflect.ParameterizedType -import java.lang.reflect.Type -import groovy.transform.CompileStatic -import com.google.common.base.Optional -import com.google.gson.* - -@CompileStatic -class GsonOptionalDeserializer implements JsonSerializer>, JsonDeserializer> { - - @Override - public Optional deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) { - final T value = context.deserialize(json, ((ParameterizedType) typeOfT).actualTypeArguments[0]) - return Optional.fromNullable(value) - } - - @Override - public JsonElement serialize(Optional src, Type typeOfSrc, JsonSerializationContext context) { - context.serialize(src.orNull(), ((ParameterizedType) typeOfSrc).actualTypeArguments[0]) - } -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/RushService.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/RushService.groovy deleted file mode 100644 index 28d71870c..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/RushService.groovy +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.rush.api - -import retrofit.http.Body -import retrofit.http.GET -import retrofit.http.POST -import retrofit.http.Path -import rx.Observable - -interface RushService { - - @POST("/ops") - Observable runScript(@Body ScriptRequest scriptRequest) - - @GET("/tasks") - Observable> listScriptDetails() - - @GET("/tasks/{scriptId}") - Observable scriptDetails(@Path("scriptId") String scriptId) - - @POST("/tasks/{scriptId}/logs") - Observable getLogs(@Path("scriptId") String scriptId, @Body ScriptRequest scriptRequest) - -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptExecution.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptExecution.groovy deleted file mode 100644 index 77a4f0fe7..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/api/ScriptExecution.groovy +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.rush.api - -import groovy.transform.CompileStatic -import groovy.transform.EqualsAndHashCode -import groovy.transform.ToString - -/** - * The details of a completed bake. - * - * @see RushService#scriptDetails - */ -@CompileStatic -@EqualsAndHashCode(includes = "id") -@ToString(includeNames = true) -class ScriptExecution { - String id - String status - String command - String image - String credentials - String container - String statusCode - String logs - Date created - Date lastUpdate -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/config/RushConfiguration.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/config/RushConfiguration.groovy deleted file mode 100644 index 26dc4c51e..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/config/RushConfiguration.groovy +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.rush.config - -import com.google.gson.* -import com.netflix.spinnaker.rosco.retrofit.RetrofitConfiguration -import com.netflix.spinnaker.rosco.rush.api.RushService -import com.netflix.spinnaker.rosco.rush.api.ScriptRequest -import groovy.transform.CompileStatic -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Import -import retrofit.Endpoint -import retrofit.Endpoints -import retrofit.RestAdapter -import retrofit.RestAdapter.LogLevel -import retrofit.client.Client -import retrofit.converter.GsonConverter - -import java.lang.reflect.Type - -@Configuration -@Import([RetrofitConfiguration]) -@CompileStatic -class RushConfiguration { - - @Autowired - Client retrofitClient - @Autowired - LogLevel retrofitLogLevel - - @Bean - Endpoint rushEndpoint(@Value('${rush.baseUrl:http://rush.prod.netflix.net}') String rushBaseUrl) { - Endpoints.newFixedEndpoint(rushBaseUrl) - } - - @Bean - RushService rushService(Endpoint rushEndpoint) { - def gson = new GsonBuilder() - .registerTypeAdapter(Date.class, new JsonDeserializer() { - Date deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - new Date(json.getAsJsonPrimitive().getAsLong()); - } - }) - .create() - - new RestAdapter.Builder() - .setEndpoint(rushEndpoint) - .setConverter(new GsonConverter(gson)) - .setClient(retrofitClient) - .setLogLevel(retrofitLogLevel) - .build() - .create(RushService) - } - - @Bean - ScriptRequest scriptRequest(@Value('${rush.credentials:null}') String credentials, @Value('${rush.image:null}') String image) { - new ScriptRequest(credentials: credentials, image: image, privileged: true) - } - -} diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/health/RushHealthIndicator.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/health/RushHealthIndicator.groovy deleted file mode 100644 index a689b7f23..000000000 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/rush/health/RushHealthIndicator.groovy +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.rush.health - -import com.netflix.spinnaker.rosco.rush.api.RushService -import groovy.transform.InheritConstructors -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.actuate.health.Health -import org.springframework.boot.actuate.health.HealthIndicator -import org.springframework.http.HttpStatus -import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Component -import org.springframework.web.bind.annotation.ResponseStatus -import retrofit.RetrofitError - -import java.util.concurrent.atomic.AtomicReference - -@Component -class RushHealthIndicator implements HealthIndicator { - - private static final Logger LOG = LoggerFactory.getLogger(RushHealthIndicator) - - @Autowired - RushService rushService - - private final AtomicReference lastException = new AtomicReference<>(null) - - @Override - Health health() { - def ex = lastException.get() - - if (ex) { - throw ex - } - - new Health.Builder().up().build() - } - - @Scheduled(initialDelay = 300000L, fixedDelay = 300000L) - void checkHealth() { - try { - // TODO(duftler): Use less-expensive health check. - rushService.listScriptDetails().toBlocking().single() - - lastException.set(null) - } catch (Exception ex) { - if (ex instanceof RetrofitError) { - ex = new RushIOException() - } - - LOG.warn "Unhealthy", ex - - lastException.set(ex) - } - } - - @ResponseStatus(value = HttpStatus.SERVICE_UNAVAILABLE, reason = "Problem communicating with Rush.") - @InheritConstructors - static class RushIOException extends RuntimeException {} -} diff --git a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/executor/BakePollerSpec.groovy b/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/executor/BakePollerSpec.groovy index 0335b49bf..9a7f9d6c5 100644 --- a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/executor/BakePollerSpec.groovy +++ b/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/executor/BakePollerSpec.groovy @@ -16,154 +16,106 @@ package com.netflix.spinnaker.rosco.executor -import com.google.gson.Gson import com.netflix.spinnaker.rosco.api.Bake import com.netflix.spinnaker.rosco.api.BakeStatus -import com.netflix.spinnaker.rosco.config.RoscoConfiguration import com.netflix.spinnaker.rosco.persistence.RedisBackedBakeStore import com.netflix.spinnaker.rosco.providers.CloudProviderBakeHandler import com.netflix.spinnaker.rosco.providers.registry.CloudProviderBakeHandlerRegistry -import com.netflix.spinnaker.rosco.rush.api.RushService -import com.netflix.spinnaker.rosco.rush.api.ScriptExecution -import com.netflix.spinnaker.rosco.rush.api.ScriptRequest -import retrofit.RetrofitError -import retrofit.client.Header -import retrofit.client.Response -import retrofit.converter.GsonConverter -import retrofit.mime.TypedByteArray -import rx.Observable +import com.netflix.spinnaker.rosco.jobs.JobExecutor import spock.lang.Specification import spock.lang.Subject import spock.lang.Unroll class BakePollerSpec extends Specification { - private static final String PACKAGE_NAME = "kato" private static final String REGION = "some-region" - private static final String SCRIPT_ID = "123" - private static final String CREDENTIALS = "some-credentials" + private static final String JOB_ID = "123" private static final String AMI_ID = "ami-3cf4a854" private static final String IMAGE_NAME = "some-image" private static final String LOGS_CONTENT = "Some logs content..." @Unroll - void 'scheduled update queries scripting engine and stores status and logs when incomplete'() { + void 'scheduled update queries job executor and stores status and logs when incomplete'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def executionStatusToBakeStateMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeStateMap) - def executionStatusToBakeResultMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeResultMap) - def scriptDetailsObservable = Observable.from(new ScriptExecution(id: SCRIPT_ID, status: executionStatus)) - def logsContentMapObservable = Observable.from([logsContent: LOGS_CONTENT]) + def jobExecutorMock = Mock(JobExecutor) + def incompleteBakeStatus = new BakeStatus(id: JOB_ID, + resource_id: JOB_ID, + state: bakeState, + result: bakeResult, + logsContent: LOGS_CONTENT) @Subject def bakePoller = new BakePoller(bakeStore: bakeStoreMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - rushService: rushServiceMock, - cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - executionStatusToBakeStateMap: executionStatusToBakeStateMapMock, - executionStatusToBakeResultMap: executionStatusToBakeResultMapMock) + executor: jobExecutorMock, + cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock) when: - bakePoller.updateBakeStatusAndLogs(SCRIPT_ID) + bakePoller.updateBakeStatusAndLogs(JOB_ID) then: - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * rushServiceMock.getLogs(SCRIPT_ID, new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME)) >> logsContentMapObservable - - 1 * executionStatusToBakeStateMapMock.convertExecutionStatusToBakeState(executionStatus) >> bakeState - 1 * executionStatusToBakeResultMapMock.convertExecutionStatusToBakeResult(executionStatus) >> bakeResult - - - 1 * bakeStoreMock.updateBakeStatus(new BakeStatus(id: SCRIPT_ID, - resource_id: SCRIPT_ID, - state: bakeState, - result: bakeResult), - [logsContent: LOGS_CONTENT]) + 1 * jobExecutorMock.updateJob(JOB_ID) >> incompleteBakeStatus + 1 * bakeStoreMock.updateBakeStatus(incompleteBakeStatus) where: - executionStatus | bakeState | bakeResult - "PREPARING" | BakeStatus.State.PENDING | null - "FETCHING_IMAGE" | BakeStatus.State.PENDING | null - "RUNNING" | BakeStatus.State.RUNNING | null - "FAILED" | BakeStatus.State.CANCELED | null + bakeState | bakeResult + BakeStatus.State.RUNNING | null + BakeStatus.State.CANCELED | BakeStatus.Result.FAILURE } - void 'scheduled update queries scripting engine and stores status, details and logs when complete'() { + void 'scheduled update queries job executor and stores status, details and logs when complete'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def executionStatusToBakeStateMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeStateMap) - def executionStatusToBakeResultMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeResultMap) - def scriptDetailsObservable = Observable.from(new ScriptExecution(id: SCRIPT_ID, status: executionStatus)) - def logsContentMapObservable = Observable.from([logsContent: "$LOGS_CONTENT\n$LOGS_CONTENT"]) + def jobExecutorMock = Mock(JobExecutor) + def completeBakeStatus = new BakeStatus(id: JOB_ID, + resource_id: JOB_ID, + state: bakeState, + result: bakeResult, + logsContent: "$LOGS_CONTENT\n$LOGS_CONTENT") + def bakeDetails = new Bake(id: JOB_ID, ami: AMI_ID, image_name: IMAGE_NAME) @Subject def bakePoller = new BakePoller(bakeStore: bakeStoreMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - rushService: rushServiceMock, - cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - executionStatusToBakeStateMap: executionStatusToBakeStateMapMock, - executionStatusToBakeResultMap: executionStatusToBakeResultMapMock) + executor: jobExecutorMock, + cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock) when: - bakePoller.updateBakeStatusAndLogs(SCRIPT_ID) + bakePoller.updateBakeStatusAndLogs(JOB_ID) then: - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * rushServiceMock.getLogs(SCRIPT_ID, new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME)) >> logsContentMapObservable - - 1 * executionStatusToBakeStateMapMock.convertExecutionStatusToBakeState(executionStatus) >> bakeState + 1 * jobExecutorMock.updateJob(JOB_ID) >> completeBakeStatus 1 * cloudProviderBakeHandlerRegistryMock.findProducer(LOGS_CONTENT) >> cloudProviderBakeHandlerMock - 1 * bakeStoreMock.retrieveRegionById(SCRIPT_ID) >> REGION - - 1 * cloudProviderBakeHandlerMock.scrapeCompletedBakeResults(REGION, SCRIPT_ID, "$LOGS_CONTENT\n$LOGS_CONTENT") >> new Bake(id: SCRIPT_ID, ami: AMI_ID, image_name: IMAGE_NAME) - 1 * bakeStoreMock.updateBakeDetails(new Bake(id: SCRIPT_ID, ami: AMI_ID, image_name: IMAGE_NAME)) - - 1 * executionStatusToBakeResultMapMock.convertExecutionStatusToBakeResult(executionStatus) >> bakeResult - 1 * bakeStoreMock.updateBakeStatus(new BakeStatus(id: SCRIPT_ID, - resource_id: SCRIPT_ID, - state: bakeState, - result: bakeResult), - [logsContent: "$LOGS_CONTENT\n$LOGS_CONTENT"]) + 1 * bakeStoreMock.retrieveRegionById(JOB_ID) >> REGION + 1 * cloudProviderBakeHandlerMock.scrapeCompletedBakeResults(REGION, JOB_ID, "$LOGS_CONTENT\n$LOGS_CONTENT") >> bakeDetails + 1 * bakeStoreMock.updateBakeDetails(bakeDetails) + 1 * bakeStoreMock.updateBakeStatus(completeBakeStatus) where: - executionStatus | bakeState | bakeResult - "SUCCESSFUL" | BakeStatus.State.COMPLETED | BakeStatus.Result.SUCCESS + bakeState | bakeResult + BakeStatus.State.COMPLETED | BakeStatus.Result.SUCCESS } - void 'scheduled update stores error and status when scripting engine throws exception'() { + void 'scheduled update stores error and status when running job status cannot be retrieved'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def executionStatusToBakeStateMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeStateMap) - def executionStatusToBakeResultMapMock = Mock(RoscoConfiguration.ExecutionStatusToBakeResultMap) - def retrofitErrorTypedInput = new TypedByteArray(null, "\"Some Rush error...\"".bytes) - def retrofitErrorResponse = new Response("http://some-rush-engine/...", 500, "Some Rush reason...", new ArrayList
(), retrofitErrorTypedInput) - def retrofitError = RetrofitError.httpError("http://some-rush-engine/...", retrofitErrorResponse, new GsonConverter(new Gson()), String.class) - - retrofitError.body + def jobExecutorMock = Mock(JobExecutor) @Subject def bakePoller = new BakePoller(bakeStore: bakeStoreMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - rushService: rushServiceMock, - cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - executionStatusToBakeStateMap: executionStatusToBakeStateMapMock, - executionStatusToBakeResultMap: executionStatusToBakeResultMapMock) + executor: jobExecutorMock, + cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock) when: - bakePoller.updateBakeStatusAndLogs(SCRIPT_ID) + bakePoller.updateBakeStatusAndLogs(JOB_ID) then: - - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> { throw retrofitError } - 1 * bakeStoreMock.storeBakeError(SCRIPT_ID, "\"Some Rush error...\"") - 1 * bakeStoreMock.cancelBakeById(SCRIPT_ID) + 1 * jobExecutorMock.updateJob(JOB_ID) >> null + 1 * bakeStoreMock.storeBakeError(JOB_ID, "Unable to retrieve status for '$JOB_ID'.") + 1 * bakeStoreMock.cancelBakeById(JOB_ID) } } diff --git a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactorySpec.groovy b/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactorySpec.groovy deleted file mode 100644 index 815b58941..000000000 --- a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/util/DockerFriendlyPackerCommandFactorySpec.groovy +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2015 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.rosco.providers.util - -import spock.lang.Shared -import spock.lang.Specification - -class DockerFriendlyPackerCommandFactorySpec extends Specification { - - @Shared - DockerFriendlyPackerCommandFactory packerCommandFactory = new DockerFriendlyPackerCommandFactory() - - @Shared - String templateFile = "some-packer-template.json" - - void "packer command is built properly with multiple parameters"() { - setup: - def parameterMap = [ - some_project_id: "some-project", - some_zone: "us-central1-a", - some_source_image: "ubuntu-1404-trusty-v20141212", - some_target_image: "some-new-image" - ] - - when: - def packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false -var some_project_id=some-project " + - "-var some_zone=us-central1-a -var some_source_image=ubuntu-1404-trusty-v20141212 " + - "-var some_target_image=some-new-image " + - "some-packer-template.json"] - } - - void "packer command is built properly with zero parameters"() { - setup: - def parameterMap = [:] - - when: - def packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false some-packer-template.json"] - } - - void "packer command elides parameters with empty keys"() { - setup: - def parameterMap = [ - "": "some-value-2", - some_key_3: "some-value-3" - ] - parameterMap[null] = "some-value-1" - - when: - def packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false -var some_key_3=some-value-3 some-packer-template.json"] - } - - void "packer command elides parameters with empty values"() { - when: - def parameterMap = [ - some_key_1: "some-value", - some_key_2: null - ] - def packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false -var some_key_1=some-value some-packer-template.json"] - - when: - parameterMap = [ - some_key_1: "some-value", - some_key_2: "" - ] - packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false -var some_key_1=some-value some-packer-template.json"] - } - - void "packer command quotes parameters with values that contain spaces"() { - when: - def parameterMap = [ - some_key_1: "some-value", - some_key_2: "some set of values" - ] - def packerCommand = packerCommandFactory.buildPackerCommand("", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "packer build -color=false -var some_key_1=some-value " + - "-var \"some_key_2=some set of values\" some-packer-template.json"] - } - - void "packer command prepends base command when specified"() { - when: - def parameterMap = [ - some_key_1: "some-value", - some_key_2: "some set of values" - ] - def packerCommand = - packerCommandFactory.buildPackerCommand("some base command ; ", parameterMap, templateFile) - - then: - packerCommand == ["sh", "-c", "some base command ; packer build -color=false -var some_key_1=some-value " + - "-var \"some_key_2=some set of values\" some-packer-template.json"] - } - -} diff --git a/rosco-web/config/packer/install_packages.sh b/rosco-web/config/packer/install_packages.sh index 9f4f502b8..b54f4fa09 100755 --- a/rosco-web/config/packer/install_packages.sh +++ b/rosco-web/config/packer/install_packages.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Make rush fail the build on errors. +# Make the build fail on errors. set -e # Strip the first part to avoid credentials leaks. diff --git a/rosco-web/config/rosco.yml b/rosco-web/config/rosco.yml index a4c415107..adbc3e96c 100644 --- a/rosco-web/config/rosco.yml +++ b/rosco-web/config/rosco.yml @@ -1,33 +1,9 @@ server: port: 8087 -rush: - baseUrl: http://localhost:8085 - # TODO(duftler): Figure out the right way to set this dynamically. +rosco: configDir: /some/path/to/config/packer -executionStatusToBakeStates: - associations: - - executionStatus: PREPARING - bakeState: PENDING - - executionStatus: FETCHING_IMAGE - bakeState: PENDING - - executionStatus: RUNNING - bakeState: RUNNING - - executionStatus: CANCELED - bakeState: CANCELED - - executionStatus: SUCCESSFUL - bakeState: COMPLETED - - executionStatus: FAILED - bakeState: CANCELED - -executionStatusToBakeResults: - associations: - - executionStatus: SUCCESSFUL - bakeResult: SUCCESS - - executionStatus: FAILED - bakeResult: FAILURE - # If a repository is set here, it will be added by packer as repository when baking images for GCE and AWS. # It is safe to leave this out (or blank) if you do not need to configure your own repository. # You can specify an apt repository (used when baking debian based images) and/or a yum repository (used when baking an @@ -251,5 +227,5 @@ swagger: description: contact: patterns: - - .*api/v1.* + - /api/v1.* - /bakeOptions.* diff --git a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy index 6108f3b6c..9ffac8975 100644 --- a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy +++ b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy @@ -21,6 +21,7 @@ import com.netflix.spinnaker.rosco.providers.azure.config.RoscoAzureConfiguratio import com.netflix.spinnaker.rosco.providers.docker.config.RoscoDockerConfiguration import com.netflix.spinnaker.rosco.providers.google.config.RoscoGoogleConfiguration import com.netflix.spinnaker.rosco.providers.openstack.config.RoscoOpenstackConfiguration +import com.netflix.spinnaker.rosco.jobs.config.LocalJobConfig import org.springframework.boot.autoconfigure.EnableAutoConfiguration import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration @@ -41,11 +42,11 @@ import javax.servlet.Filter "com.netflix.spinnaker.rosco.controllers", "com.netflix.spinnaker.rosco.executor", "com.netflix.spinnaker.rosco.filters", + "com.netflix.spinnaker.rosco.jobs", "com.netflix.spinnaker.rosco.persistence", - "com.netflix.spinnaker.rosco.rush", "com.netflix.spinnaker.config" ]) -@Import([RoscoAWSConfiguration, RoscoAzureConfiguration, RoscoDockerConfiguration, RoscoGoogleConfiguration, RoscoOpenstackConfiguration]) +@Import([RoscoAWSConfiguration, RoscoAzureConfiguration, RoscoDockerConfiguration, RoscoGoogleConfiguration, RoscoOpenstackConfiguration, LocalJobConfig]) @EnableAutoConfiguration(exclude = [BatchAutoConfiguration, GroovyTemplateAutoConfiguration]) @EnableScheduling class Main extends SpringBootServletInitializer { diff --git a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/controllers/BakeryController.groovy b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/controllers/BakeryController.groovy index fc216d715..93d360980 100644 --- a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/controllers/BakeryController.groovy +++ b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/controllers/BakeryController.groovy @@ -23,9 +23,8 @@ import com.netflix.spinnaker.rosco.api.BakeStatus import com.netflix.spinnaker.rosco.persistence.BakeStore import com.netflix.spinnaker.rosco.providers.CloudProviderBakeHandler import com.netflix.spinnaker.rosco.providers.registry.CloudProviderBakeHandlerRegistry -import com.netflix.spinnaker.rosco.rush.api.RushService -import com.netflix.spinnaker.rosco.rush.api.ScriptExecution -import com.netflix.spinnaker.rosco.rush.api.ScriptRequest +import com.netflix.spinnaker.rosco.jobs.JobExecutor +import com.netflix.spinnaker.rosco.jobs.JobRequest import groovy.util.logging.Slf4j import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -49,10 +48,7 @@ class BakeryController { BakeStore bakeStore @Autowired - ScriptRequest baseScriptRequest - - @Autowired - RushService rushService + JobExecutor jobExecutor @Autowired CloudProviderBakeHandlerRegistry cloudProviderBakeHandlerRegistry @@ -91,6 +87,38 @@ class BakeryController { [error: "bake.options.not.found", status: HttpStatus.NOT_FOUND, messages: ["Bake options not found. " + e.message]] } + private BakeStatus runBake(String bakeKey, String region, BakeRequest bakeRequest, JobRequest jobRequest) { + String jobId = jobExecutor.startJob(jobRequest) + + // Give the job jobExecutor some time to kick off the job. + // The goal here is to fail fast. If it takes too much time, no point in waiting here. + sleep(1000) + + // Update the status right away so we can fail fast if necessary. + BakeStatus newBakeStatus = jobExecutor.updateJob(jobId) + + if (newBakeStatus.result == BakeStatus.Result.FAILURE && newBakeStatus.logsContent) { + throw new IllegalArgumentException(newBakeStatus.logsContent) + + // If we don't have logs content to return here, just let the poller try again on the next iteration. + } + + // Ok, it didn't fail right away; the bake is underway. + BakeStatus returnedBakeStatus = bakeStore.storeNewBakeStatus(bakeKey, + region, + bakeRequest, + newBakeStatus, + jobRequest.tokenizedCommand.join(" ")) + + // Check if the script returned a bake status set by the winner of a race. + if (returnedBakeStatus.id != newBakeStatus.id) { + // Kill the new sub-process. + jobExecutor.cancelJob(newBakeStatus.id) + } + + return returnedBakeStatus + } + @RequestMapping(value = '/api/v1/{region}/bake', method = RequestMethod.POST) BakeStatus createBake(@PathVariable("region") String region, @RequestBody BakeRequest bakeRequest, @@ -116,10 +144,10 @@ class BakeryController { } def packerCommand = cloudProviderBakeHandler.producePackerCommand(region, bakeRequest) - def scriptRequest = baseScriptRequest.copyWith(tokenizedCommand: packerCommand) + def jobRequest = new JobRequest(tokenizedCommand: packerCommand) if (bakeStore.acquireBakeLock(bakeKey)) { - return runBake(bakeKey, region, bakeRequest, scriptRequest) + return runBake(bakeKey, region, bakeRequest, jobRequest) } else { def startTime = System.currentTimeMillis() @@ -136,7 +164,7 @@ class BakeryController { // Maybe the TTL expired but the bake status wasn't set for some other reason? Let's try again before giving up. if (bakeStore.acquireBakeLock(bakeKey)) { - return runBake(bakeKey, region, bakeRequest, scriptRequest) + return runBake(bakeKey, region, bakeRequest, jobRequest) } throw new IllegalArgumentException("Unable to acquire lock and unable to determine id of lock holder for bake " + @@ -147,31 +175,10 @@ class BakeryController { } } - private BakeStatus runBake(String bakeKey, String region, BakeRequest bakeRequest, ScriptRequest scriptRequest) { - def scriptId = rushService.runScript(scriptRequest).toBlocking().single() - - // Give the script engine some time to kick off the execution. - // The goal here is to fail fast. If it takes too much time, no point in waiting here. - sleep(1000) - - ScriptExecution scriptExecution = rushService.scriptDetails(scriptId.id).toBlocking().single() - - if (scriptExecution.status == "FAILED") { - def logs = rushService.getLogs(scriptId.id, scriptRequest).toBlocking().single() - - if (logs?.logsContent) { - throw new IllegalArgumentException(logs.logsContent) - } - - // If we don't have logs content to return here, just let the poller try again on the next iteration. - } - - // Ok, it didn't fail right away. The bake is underway. - return bakeStore.storeNewBakeStatus(bakeKey, region, bakeRequest, scriptId.id) - } - + @ApiOperation(value = "Look up bake request status") @RequestMapping(value = "/api/v1/{region}/status/{statusId}", method = RequestMethod.GET) - BakeStatus lookupStatus(@PathVariable("region") String region, @PathVariable("statusId") String statusId) { + BakeStatus lookupStatus(@ApiParam(value = "The region of the bake request to lookup", required = true) @PathVariable("region") String region, + @ApiParam(value = "The id of the bake request to lookup", required = true) @PathVariable("statusId") String statusId) { def bakeStatus = bakeStore.retrieveBakeStatusById(statusId) if (bakeStatus) { @@ -181,7 +188,7 @@ class BakeryController { throw new IllegalArgumentException("Unable to retrieve status for '$statusId'.") } - @ApiOperation(value = "Look up a bake", notes = "Some longer description of looking up a bake.") + @ApiOperation(value = "Look up bake details") @RequestMapping(value = "/api/v1/{region}/bake/{bakeId}", method = RequestMethod.GET) Bake lookupBake(@ApiParam(value = "The region of the bake to lookup", required = true) @PathVariable("region") String region, @ApiParam(value = "The id of the bake to lookup", required = true) @PathVariable("bakeId") String bakeId) { @@ -202,7 +209,6 @@ class BakeryController { Map logsContentMap = bakeStore.retrieveBakeLogsById(statusId) if (logsContentMap?.logsContent) { - return html ? "
$logsContentMap.logsContent
" : logsContentMap.logsContent } @@ -211,7 +217,8 @@ class BakeryController { // TODO(duftler): Synchronize this with existing bakery api. @RequestMapping(value = '/api/v1/{region}/bake', method = RequestMethod.DELETE) - String deleteBake(@PathVariable("region") String region, @RequestBody BakeRequest bakeRequest) { + String deleteBake(@PathVariable("region") String region, + @RequestBody BakeRequest bakeRequest) { if (!bakeRequest.cloud_provider_type) { bakeRequest = bakeRequest.copyWith(cloud_provider_type: defaultCloudProviderType) } @@ -232,14 +239,16 @@ class BakeryController { } // TODO(duftler): Synchronize this with existing bakery api. + @ApiOperation(value = "Cancel bake request") @RequestMapping(value = "/api/v1/{region}/cancel/{statusId}", method = RequestMethod.GET) - String cancelBake(@PathVariable("region") String region, @PathVariable("statusId") String statusId) { + String cancelBake(@ApiParam(value = "The region of the bake request to cancel", required = true) @PathVariable("region") String region, + @ApiParam(value = "The id of the bake request to cancel", required = true) @PathVariable("statusId") String statusId) { if (bakeStore.cancelBakeById(statusId)) { + jobExecutor.cancelJob(statusId) + return "Canceled bake '$statusId'." } - // TODO(duftler): Instruct the scripting engine to kill the execution. - throw new IllegalArgumentException("Unable to locate incomplete bake with id '$statusId'.") } @@ -248,7 +257,7 @@ class BakeryController { if (!bakeStatus) { return null - } else if (bakeStatus.state == BakeStatus.State.PENDING || bakeStatus.state == BakeStatus.State.RUNNING) { + } else if (bakeStatus.state == BakeStatus.State.RUNNING) { return bakeStatus } else if (bakeStatus.state == BakeStatus.State.COMPLETED && bakeStatus.result == BakeStatus.Result.SUCCESS) { return bakeStatus diff --git a/rosco-web/src/test/groovy/com/netflix/spinnaker/rosco/controllers/BakeryControllerSpec.groovy b/rosco-web/src/test/groovy/com/netflix/spinnaker/rosco/controllers/BakeryControllerSpec.groovy index b63968007..72923c0b8 100644 --- a/rosco-web/src/test/groovy/com/netflix/spinnaker/rosco/controllers/BakeryControllerSpec.groovy +++ b/rosco-web/src/test/groovy/com/netflix/spinnaker/rosco/controllers/BakeryControllerSpec.groovy @@ -24,81 +24,75 @@ import com.netflix.spinnaker.rosco.persistence.RedisBackedBakeStore import com.netflix.spinnaker.rosco.providers.CloudProviderBakeHandler import com.netflix.spinnaker.rosco.providers.registry.CloudProviderBakeHandlerRegistry import com.netflix.spinnaker.rosco.providers.registry.DefaultCloudProviderBakeHandlerRegistry -import com.netflix.spinnaker.rosco.rush.api.RushService -import com.netflix.spinnaker.rosco.rush.api.ScriptExecution -import com.netflix.spinnaker.rosco.rush.api.ScriptId -import com.netflix.spinnaker.rosco.rush.api.ScriptRequest -import rx.Observable +import com.netflix.spinnaker.rosco.jobs.JobExecutor +import com.netflix.spinnaker.rosco.jobs.JobRequest import spock.lang.Specification import spock.lang.Subject -import spock.lang.Unroll class BakeryControllerSpec extends Specification { private static final String PACKAGE_NAME = "kato" private static final String REGION = "some-region" - private static final String SCRIPT_ID = "123" - private static final String EXISTING_SCRIPT_ID = "456" - private static final String CREDENTIALS = "some-credentials" + private static final String JOB_ID = "123" + private static final String EXISTING_JOB_ID = "456" private static final String AMI_ID = "ami-3cf4a854" private static final String IMAGE_NAME = "some-image" private static final String BAKE_KEY = "bake:gce:ubuntu:$PACKAGE_NAME" + private static final String PACKER_COMMAND = "packer build ..." private static final String LOGS_CONTENT = "Some logs content..." - void 'create bake issues script command and returns new status'() { + void 'create bake launches job and returns new status'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "RUNNING")) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def runningBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."])) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> runningBakeStatus + 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, runningBakeStatus, PACKER_COMMAND) >> runningBakeStatus + returnedBakeStatus == runningBakeStatus } - void 'create bake fails fast if script engine returns FAILED'() { + void 'create bake fails fast if job executor returns CANCELED'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def scriptRequest = new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."]) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "FAILED")) - def getLogsObservable = Observable.from([logsContent: "Some kind of failure..."]) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def failedBakeStatus = new BakeStatus(id: JOB_ID, + resource_id: JOB_ID, + state: BakeStatus.State.CANCELED, + result: BakeStatus.Result.FAILURE, + logsContent: "Some kind of failure...") @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: bakeryController.createBake(REGION, bakeRequest, null) @@ -107,11 +101,10 @@ class BakeryControllerSpec extends Specification { 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(scriptRequest) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * rushServiceMock.getLogs(SCRIPT_ID, scriptRequest) >> getLogsObservable + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> failedBakeStatus IllegalArgumentException e = thrown() e.message == "Some kind of failure..." } @@ -121,30 +114,28 @@ class BakeryControllerSpec extends Specification { def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def runningBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - bakeStore: bakeStoreMock, - rushService: rushServiceMock) + bakeStore: bakeStoreMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> false 4 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) + 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> runningBakeStatus + returnedBakeStatus == runningBakeStatus } void 'create bake polls for status when lock cannot be acquired, but tries for lock again if status cannot be obtained'() { @@ -152,35 +143,33 @@ class BakeryControllerSpec extends Specification { def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "RUNNING")) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def runningBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> false (10.._) * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."])) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> runningBakeStatus + 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, runningBakeStatus, PACKER_COMMAND) >> runningBakeStatus + returnedBakeStatus == new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) } void 'create bake polls for status when lock cannot be acquired, tries for lock again if status cannot be obtained, and throws exception if that fails'() { @@ -188,7 +177,6 @@ class BakeryControllerSpec extends Specification { def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", @@ -196,9 +184,7 @@ class BakeryControllerSpec extends Specification { @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - bakeStore: bakeStoreMock, - rushService: rushServiceMock) + bakeStore: bakeStoreMock) when: bakeryController.createBake(REGION, bakeRequest, null) @@ -207,7 +193,7 @@ class BakeryControllerSpec extends Specification { 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> false (10.._) * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> null 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> false @@ -235,8 +221,7 @@ class BakeryControllerSpec extends Specification { e.message == "Unknown provider type 'gce'." } - @Unroll - void 'create bake returns existing status when prior bake is pending or running'() { + void 'create bake returns existing status when prior bake is running'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) @@ -245,22 +230,22 @@ class BakeryControllerSpec extends Specification { package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def runningBakeStatus = new BakeStatus(id: EXISTING_JOB_ID, + resource_id: EXISTING_JOB_ID, + state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, bakeStore: bakeStoreMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY - 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: bakeState) - bakeStatus == new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: bakeState) - - where: - bakeState << [BakeStatus.State.PENDING, BakeStatus.State.RUNNING] + 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> runningBakeStatus + returnedBakeStatus == runningBakeStatus } void 'create bake returns existing status when prior bake is completed and successful'() { @@ -272,145 +257,150 @@ class BakeryControllerSpec extends Specification { package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def completedBakeStatus = new BakeStatus(id: EXISTING_JOB_ID, + resource_id: EXISTING_JOB_ID, + state: BakeStatus.State.COMPLETED, + result: BakeStatus.Result.SUCCESS) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, bakeStore: bakeStoreMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY - 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: BakeStatus.State.COMPLETED, result: BakeStatus.Result.SUCCESS) - bakeStatus == new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: BakeStatus.State.COMPLETED, result: BakeStatus.Result.SUCCESS) + 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> completedBakeStatus + returnedBakeStatus == completedBakeStatus } - void 'create bake issues script command and returns new status when prior bake is completed and failure'() { + void 'create bake launches job and returns new status when prior bake is completed and failure'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "RUNNING")) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def failedBakeStatus = new BakeStatus(id: EXISTING_JOB_ID, + resource_id: EXISTING_JOB_ID, + state: BakeStatus.State.CANCELED, + result: BakeStatus.Result.FAILURE) + def newBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY - 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: BakeStatus.State.COMPLETED, result: BakeStatus.Result.FAILURE) - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> failedBakeStatus + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."])) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> newBakeStatus + 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, newBakeStatus, PACKER_COMMAND) >> newBakeStatus + returnedBakeStatus == newBakeStatus } - @Unroll - void 'create bake issues script command and returns new status when prior bake is suspended or canceled'() { + void 'create bake launches job and returns new status when prior bake is canceled'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "RUNNING")) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def canceledBakeStatus = new BakeStatus(id: EXISTING_JOB_ID, + resource_id: EXISTING_JOB_ID, + state: BakeStatus.State.CANCELED) + def newBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, null) + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, null) then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY - 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> new BakeStatus(id: EXISTING_SCRIPT_ID, resource_id: EXISTING_SCRIPT_ID, state: bakeState) - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * bakeStoreMock.retrieveBakeStatusByKey(BAKE_KEY) >> canceledBakeStatus + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."])) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - - where: - bakeState << [BakeStatus.State.SUSPENDED, BakeStatus.State.CANCELED] + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> newBakeStatus + 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, newBakeStatus, PACKER_COMMAND) >> newBakeStatus + returnedBakeStatus == newBakeStatus } - void 'create bake with rebake deletes existing status, issues script command and returns new status no matter the pre-existing status'() { + void 'create bake with rebake deletes existing status, launches job and returns new status no matter the pre-existing status'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def cloudProviderBakeHandlerMock = Mock(CloudProviderBakeHandler) def bakeStoreMock = Mock(RedisBackedBakeStore) - def rushServiceMock = Mock(RushService) - def runScriptObservable = Observable.from(new ScriptId(id: SCRIPT_ID)) - def scriptDetailsObservable = Observable.from(new ScriptExecution(status: "RUNNING")) + def jobExecutorMock = Mock(JobExecutor) def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGE_NAME, base_os: "ubuntu", cloud_provider_type: BakeRequest.CloudProviderType.gce) + def newBakeStatus = new BakeStatus(id: JOB_ID, resource_id: JOB_ID, state: BakeStatus.State.RUNNING) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock, - rushService: rushServiceMock) + jobExecutor: jobExecutorMock) when: - def bakeStatus = bakeryController.createBake(REGION, bakeRequest, "1") + def returnedBakeStatus = bakeryController.createBake(REGION, bakeRequest, "1") then: 1 * cloudProviderBakeHandlerRegistryMock.lookup(BakeRequest.CloudProviderType.gce) >> cloudProviderBakeHandlerMock 1 * cloudProviderBakeHandlerMock.produceBakeKey(REGION, bakeRequest) >> BAKE_KEY 1 * bakeStoreMock.deleteBakeByKey(BAKE_KEY) - 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> ["packer build ..."] + 1 * cloudProviderBakeHandlerMock.producePackerCommand(REGION, bakeRequest) >> [PACKER_COMMAND] 1 * bakeStoreMock.acquireBakeLock(BAKE_KEY) >> true - 1 * rushServiceMock.runScript(new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME, tokenizedCommand: ["packer build ..."])) >> runScriptObservable - 1 * rushServiceMock.scriptDetails(SCRIPT_ID) >> scriptDetailsObservable - 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING) + 1 * jobExecutorMock.startJob(new JobRequest(tokenizedCommand: [PACKER_COMMAND])) >> JOB_ID + 1 * jobExecutorMock.updateJob(JOB_ID) >> newBakeStatus + 1 * bakeStoreMock.storeNewBakeStatus(BAKE_KEY, REGION, bakeRequest, newBakeStatus, PACKER_COMMAND) >> newBakeStatus + returnedBakeStatus == newBakeStatus } void 'lookup status queries bake store and returns bake status'() { setup: def cloudProviderBakeHandlerRegistryMock = Mock(CloudProviderBakeHandlerRegistry) def bakeStoreMock = Mock(RedisBackedBakeStore) + def runningBakeStatus = new BakeStatus(id: JOB_ID, + resource_id: JOB_ID, + state: BakeStatus.State.RUNNING, + result: null) @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, bakeStore: bakeStoreMock) when: - def bakeStatus = bakeryController.lookupStatus(REGION, SCRIPT_ID) + def returnedBakeStatus = bakeryController.lookupStatus(REGION, JOB_ID) then: - 1 * bakeStoreMock.retrieveBakeStatusById(SCRIPT_ID) >> new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING, result: null) - bakeStatus == new BakeStatus(id: SCRIPT_ID, resource_id: SCRIPT_ID, state: BakeStatus.State.PENDING, result: null) + 1 * bakeStoreMock.retrieveBakeStatusById(JOB_ID) >> runningBakeStatus + returnedBakeStatus == runningBakeStatus } - void 'lookup status throws exception when script execution cannot be found'() { + void 'lookup status throws exception when job cannot be found'() { setup: def bakeStoreMock = Mock(RedisBackedBakeStore) @@ -418,10 +408,10 @@ class BakeryControllerSpec extends Specification { def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - bakeryController.lookupStatus(REGION, SCRIPT_ID) + bakeryController.lookupStatus(REGION, JOB_ID) then: - 1 * bakeStoreMock.retrieveBakeStatusById(SCRIPT_ID) >> null + 1 * bakeStoreMock.retrieveBakeStatusById(JOB_ID) >> null IllegalArgumentException e = thrown() e.message == "Unable to retrieve status for '123'." } @@ -431,30 +421,28 @@ class BakeryControllerSpec extends Specification { def bakeStoreMock = Mock(RedisBackedBakeStore) @Subject - def bakeryController = new BakeryController(baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - bakeStore: bakeStoreMock) + def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - def bakeDetails = bakeryController.lookupBake(REGION, SCRIPT_ID) + def bakeDetails = bakeryController.lookupBake(REGION, JOB_ID) then: - 1 * bakeStoreMock.retrieveBakeDetailsById(SCRIPT_ID) >> new Bake(id: SCRIPT_ID, ami: AMI_ID, image_name: IMAGE_NAME) - bakeDetails == new Bake(id: SCRIPT_ID, ami: AMI_ID, image_name: IMAGE_NAME) + 1 * bakeStoreMock.retrieveBakeDetailsById(JOB_ID) >> new Bake(id: JOB_ID, ami: AMI_ID, image_name: IMAGE_NAME) + bakeDetails == new Bake(id: JOB_ID, ami: AMI_ID, image_name: IMAGE_NAME) } - void 'lookup bake throws exception when script execution cannot be found'() { + void 'lookup bake throws exception when job cannot be found'() { setup: def bakeStoreMock = Mock(RedisBackedBakeStore) @Subject - def bakeryController = new BakeryController(baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), - bakeStore: bakeStoreMock) + def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - bakeryController.lookupBake(REGION, SCRIPT_ID) + bakeryController.lookupBake(REGION, JOB_ID) then: - 1 * bakeStoreMock.retrieveBakeDetailsById(SCRIPT_ID) >> null + 1 * bakeStoreMock.retrieveBakeDetailsById(JOB_ID) >> null IllegalArgumentException e = thrown() e.message == "Unable to retrieve bake details for '123'." } @@ -467,14 +455,14 @@ class BakeryControllerSpec extends Specification { def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - def response = bakeryController.lookupLogs(REGION, SCRIPT_ID, false) + def response = bakeryController.lookupLogs(REGION, JOB_ID, false) then: - 1 * bakeStoreMock.retrieveBakeLogsById(SCRIPT_ID) >> [logsContent: LOGS_CONTENT] + 1 * bakeStoreMock.retrieveBakeLogsById(JOB_ID) >> [logsContent: LOGS_CONTENT] response == LOGS_CONTENT } - void 'lookup logs throws exception when script execution logs are empty or malformed'() { + void 'lookup logs throws exception when job logs are empty or malformed'() { setup: def bakeStoreMock = Mock(RedisBackedBakeStore) @@ -482,34 +470,34 @@ class BakeryControllerSpec extends Specification { def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - bakeryController.lookupLogs(REGION, SCRIPT_ID, false) + bakeryController.lookupLogs(REGION, JOB_ID, false) then: - 1 * bakeStoreMock.retrieveBakeLogsById(SCRIPT_ID) >> null + 1 * bakeStoreMock.retrieveBakeLogsById(JOB_ID) >> null IllegalArgumentException e = thrown() e.message == "Unable to retrieve logs for '123'." when: - bakeryController.lookupLogs(REGION, SCRIPT_ID, false) + bakeryController.lookupLogs(REGION, JOB_ID, false) then: - 1 * bakeStoreMock.retrieveBakeLogsById(SCRIPT_ID) >> [:] + 1 * bakeStoreMock.retrieveBakeLogsById(JOB_ID) >> [:] e = thrown() e.message == "Unable to retrieve logs for '123'." when: - bakeryController.lookupLogs(REGION, SCRIPT_ID, false) + bakeryController.lookupLogs(REGION, JOB_ID, false) then: - 1 * bakeStoreMock.retrieveBakeLogsById(SCRIPT_ID) >> [logsContent: null] + 1 * bakeStoreMock.retrieveBakeLogsById(JOB_ID) >> [logsContent: null] e = thrown() e.message == "Unable to retrieve logs for '123'." when: - bakeryController.lookupLogs(REGION, SCRIPT_ID, false) + bakeryController.lookupLogs(REGION, JOB_ID, false) then: - 1 * bakeStoreMock.retrieveBakeLogsById(SCRIPT_ID) >> [logsContent: ''] + 1 * bakeStoreMock.retrieveBakeLogsById(JOB_ID) >> [logsContent: ''] e = thrown() e.message == "Unable to retrieve logs for '123'." } @@ -526,7 +514,6 @@ class BakeryControllerSpec extends Specification { @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock) when: @@ -551,7 +538,6 @@ class BakeryControllerSpec extends Specification { @Subject def bakeryController = new BakeryController(cloudProviderBakeHandlerRegistry: cloudProviderBakeHandlerRegistryMock, - baseScriptRequest: new ScriptRequest(credentials: CREDENTIALS, image: IMAGE_NAME), bakeStore: bakeStoreMock) when: @@ -565,19 +551,22 @@ class BakeryControllerSpec extends Specification { e.message == "Unable to locate bake with key '$BAKE_KEY'." } - void 'cancel bake updates bake store and returns status'() { + void 'cancel bake updates bake store, kills the job and returns status'() { setup: def bakeStoreMock = Mock(RedisBackedBakeStore) + def jobExecutorMock = Mock(JobExecutor) - @Subject - def bakeryController = new BakeryController(bakeStore: bakeStoreMock) + @Subject + def bakeryController = new BakeryController(bakeStore: bakeStoreMock, + jobExecutor: jobExecutorMock) when: - def response = bakeryController.cancelBake(REGION, SCRIPT_ID) + def response = bakeryController.cancelBake(REGION, JOB_ID) then: - 1 * bakeStoreMock.cancelBakeById(SCRIPT_ID) >> true - response == "Canceled bake '$SCRIPT_ID'." + 1 * bakeStoreMock.cancelBakeById(JOB_ID) >> true + 1 * jobExecutorMock.cancelJob(JOB_ID) + response == "Canceled bake '$JOB_ID'." } void 'cancel bake throws exception when bake id cannot be found'() { @@ -588,12 +577,12 @@ class BakeryControllerSpec extends Specification { def bakeryController = new BakeryController(bakeStore: bakeStoreMock) when: - bakeryController.cancelBake(REGION, SCRIPT_ID) + bakeryController.cancelBake(REGION, JOB_ID) then: - 1 * bakeStoreMock.cancelBakeById(SCRIPT_ID) >> false + 1 * bakeStoreMock.cancelBakeById(JOB_ID) >> false IllegalArgumentException e = thrown() - e.message == "Unable to locate incomplete bake with id '$SCRIPT_ID'." + e.message == "Unable to locate incomplete bake with id '$JOB_ID'." } def "should list bake options by cloud provider"() {