forked from openedx-unsupported/devstack
-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathMakefile
459 lines (339 loc) · 22.4 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
########################################################################################################################
#
# When adding a new target:
# - If you are adding a new service make sure the dev.reset target will fully reset said service.
#
########################################################################################################################
.DEFAULT_GOAL := help
.PHONY: analytics-pipeline-devstack-test analytics-pipeline-shell \
analyticspipeline-shell backup build-courses check-memory \
create-test-course credentials-shell destroy dev.cache-programs \
dev.check dev.checkout dev.clone dev.clone.ssh dev.nfs.setup \
devpi-password dev.provision dev.provision.analytics_pipeline \
dev.provision.services dev.provision.xqueue dev.pull dev.repo.reset \
dev.reset dev.status dev.sync.daemon.start dev.sync.provision \
dev.sync.requirements dev.sync.up dev.up dev.up.all \
dev.up.analytics_pipeline dev.up.watchers dev.up.with-programs \
discovery-shell down e2e-shell e2e-tests ecommerce-shell \
feature-toggle-state forum-restart-devserver healthchecks help \
lms-restart lms-shell lms-static lms-update-db lms-watcher-shell logs \
mongo-shell mysql-shell mysql-shell-edxapp provision pull \
pull.analytics_pipeline pull.xqueue registrar-shell requirements \
restore selfcheck static stats stop stop.all stop.analytics_pipeline \
stop.watchers stop.xqueue studio-restart studio-shell studio-static \
studio-update-db studio-watcher-shell update-db upgrade upgrade \
validate validate-lms-volume vnc-passwords xqueue_consumer-restart \
xqueue_consumer-shell xqueue-restart xqueue-shell
# Include options (configurable through options.local.mk)
include options.mk
# Include local overrides to options.
# You can use this file to configure your Devstack. It is ignored by git.
-include options.local.mk # Prefix with hyphen to tolerate absence of file.
# Include local makefile with additional targets.
-include local.mk # Prefix with hyphen to tolerate absence of file.
# Docker Compose YAML files to define services and their volumes.
# Depending on the value of FS_SYNC_STRATEGY, we use a slightly different set of
# files, enabling use of different strategies to synchronize files between the host and
# the containers.
# Some services are only available for certain values of FS_SYNC_STRATEGY.
# For example, the LMS/Studio asset watchers are only available for local-mounts and nfs,
# and XQueue and the Analytics Pipeline are only available for local-mounts.
ifeq ($(FS_SYNC_STRATEGY),local-mounts)
DOCKER_COMPOSE_FILES := \
-f docker-compose-host.yml \
-f docker-compose-themes.yml \
-f docker-compose-watchers.yml \
-f docker-compose-xqueue.yml \
-f docker-compose-analytics-pipeline.yml \
-f docker-compose-marketing-site.yml
endif
ifeq ($(FS_SYNC_STRATEGY),nfs)
DOCKER_COMPOSE_FILES := \
-f docker-compose-host-nfs.yml \
-f docker-compose-themes-nfs.yml \
-f docker-compose-watchers-nfs.yml
endif
ifeq ($(FS_SYNC_STRATEGY),docker-sync)
DOCKER_COMPOSE_FILES := \
-f docker-compose-sync.yml
endif
ifndef DOCKER_COMPOSE_FILES
$(error FS_SYNC_STRATEGY is set to $(FS_SYNC_STRATEGY). Must be one of: local-mounts, nfs, docker-sync)
endif
# All three filesystem synchronization strategy require the main docker-compose.yml file.
DOCKER_COMPOSE_FILES := -f docker-compose.yml $(DOCKER_COMPOSE_FILES)
OS := $(shell uname)
# Need to run some things under winpty in a Windows git-bash shell
# (but not when calling bash from a command shell or PowerShell)
ifneq (,$(MINGW_PREFIX))
WINPTY := winpty
else
WINPTY :=
endif
# Don't try redirecting to /dev/null in any Windows shell
ifneq (,$(findstring MINGW,$(OS)))
DEVNULL :=
else
DEVNULL := >/dev/null
endif
# Include specialized Make commands.
include marketing.mk
# Include Tahoe commands
include tahoe.mk
# Export Makefile variables to recipe shells.
export
# Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
help: ## Display this help message
@echo "Please use \`make <target>' where <target> is one of"
@awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
requirements: ## Install requirements
pip install -r requirements/base.txt
upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
upgrade: ## Upgrade requirements with pip-tools
pip install -qr requirements/pip-tools.txt
pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
pip-compile --upgrade -o requirements/base.txt requirements/base.in
dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if set, "master" otherwise
./repo.sh checkout
dev.clone: ## Clone service repos using HTTPS method to the parent directory
./repo.sh clone
dev.clone.ssh: ## Clone service repos using SSH method to the parent directory
./repo.sh clone_ssh
dev.provision.services: ## Provision default services with local mounted directories
# We provision all default services as well as 'e2e' (end-to-end tests).
# e2e is not part of `DEFAULT_SERVICES` because it isn't a service;
# it's just a way to tell ./provision.sh that the fake data for end-to-end
# tests should be prepared.
$(WINPTY) bash ./provision.sh $(DEFAULT_SERVICES)+e2e
dev.provision.services.%: ## Provision specified services with local mounted directories, separated by plus signs
$(WINPTY) bash ./provision.sh $*
dev.provision: check-memory dev.clone.ssh dev.provision.services ## Provision dev environment with default services, and then stop them.
make dev.up
make tahoe.provision
make amc.provision
make stop
dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS.
$(WINPTY) bash ./programs/provision.sh cache
dev.provision.xqueue: dev.provision.services.xqueue
dev.reset: down dev.repo.reset pull dev.up static update-db ## Attempts to reset the local devstack to the master working state
dev.status: ## Prints the status of all git repositories
$(WINPTY) bash ./repo.sh status
dev.repo.reset: ## Attempts to reset the local repo checkouts to the master working state
$(WINPTY) bash ./repo.sh reset
dev.pull: dev.pull.$(DEFAULT_SERVICES) ## Pull Docker images required by default services.
dev.pull.without-deps.%: ## Pull latest Docker images for services (separated by plus-signs).
docker-compose $(DOCKER_COMPOSE_FILES) pull $$(echo $* | tr + " ")
dev.pull.%: ## Pull latest Docker images for services (separated by plus-signs) and all their dependencies.
docker-compose $(DOCKER_COMPOSE_FILES) pull --include-deps $$(echo $* | tr + " ")
dev.up: dev.up.$(DEFAULT_SERVICES) check-memory ## Bring up default services.
@sleep 1
make tahoe.provision
make tahoe.chown
dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes.
docker-compose $(DOCKER_COMPOSE_FILES) up -d $$(echo $* | tr + " ")
ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
make dev.cache-programs
endif
dev.up.without-deps.%: ## Bring up specific services (separated by plus-signs) without dependencies.
docker-compose $(DOCKER_COMPOSE_FILES) up --d --no-deps $$(echo $* | tr + " ")
dev.up.with-programs: dev.up dev.cache-programs ## Bring up a all services and cache programs in LMS.
dev.up.with-programs.%: dev.up.$* dev.cache-programs ## Bring up a service and its dependencies and cache programs in LMS.
dev.up.watchers: check-memory dev.up.lms_watcher+studio_watcher ## Bring up asset watcher containers
dev.nfs.setup: ## Sets up an nfs server on the /Users folder, allowing nfs mounting on docker
./setup_native_nfs_docker_osx.sh
dev.nfs.%:
FS_SYNC_STRATEGY=nfs make dev.$*
dev.up.all: dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers
dev.print-container.%: ## Get the ID of the running container for a given service.
@# Can be run as ``make --silent dev.print-container.$service`` for just ID.
@echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*)
# TODO: Improve or rip out Docker Sync targets.
# They are not well-fleshed-out and it is not clear if anyone uses them.
dev.sync.daemon.start: ## Start the docker-sycn daemon
docker-sync start
dev.sync.provision: dev.sync.daemon.start ## Provision with docker-sync enabled
FS_SYNC_STRATEGY=docker-sync make dev.provision
dev.sync.requirements: ## Install requirements
gem install docker-sync
dev.sync.up: dev.sync.daemon.start ## Bring up all services with docker-sync enabled
FS_SYNC_STRATEGY=docker-sync make dev.up
dev.check: dev.check.$(DEFAULT_SERVICES) ## Run checks for the default service set.
dev.check.%: # Run checks for a given service or set of services (separated by plus-signs).
$(WINPTY) bash ./check.sh $*
provision: | dev.provision ## This command will be deprecated in a future release, use dev.provision
echo "\033[0;31mThis command will be deprecated in a future release, use dev.provision\033[0m"
stop: ## Stop all services
(test -d .docker-sync && docker-sync stop) || true ## Ignore failure here
docker-compose $(DOCKER_COMPOSE_FILES) stop
stop.watchers: ## Stop asset watchers
docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher
stop.all: | stop.analytics_pipeline stop stop.watchers ## Stop all containers, including asset watchers
stop.xqueue: ## Stop the XQueue and XQueue-Consumer containers
docker-compose $(DOCKER_COMPOSE_FILES) stop xqueue xqueue_consumer
down: ## Remove all service containers and networks
(test -d .docker-sync && docker-sync clean) || true ## Ignore failure here
docker-compose $(DOCKER_COMPOSE_FILES) down
destroy: ## Remove all devstack-related containers, networks, and volumes
$(WINPTY) bash ./destroy.sh
logs: ## View logs from containers running in detached mode
docker-compose $(DOCKER_COMPOSE_FILES) logs -f
%-logs: ## View the logs of the specified service container
docker-compose $(DOCKER_COMPOSE_FILES) logs -f --tail=500 $*
RED="\033[0;31m"
YELLOW="\033[0;33m"
GREY="\033[1;90m"
NO_COLOR="\033[0m"
pull: dev.pull
@echo -n $(RED)
@echo "******************* PLEASE NOTE ********************************"
@echo -n $(YELLOW)
@echo "The 'make pull' command is deprecated."
@echo "Please use 'make dev.pull.<service>'."
@echo "It will pull all the images that the given serivce depends upon."
@echo "Example: "
@echo "----------------------------------"
@echo -n $(GREY)
@echo "~/devstack$$ make dev.pull.lms"
@echo " Pulling chrome ... done"
@echo " Pulling firefox ... done"
@echo " Pulling memcached ... done"
@echo " Pulling mongo ... done"
@echo " Pulling mysql ... done"
@echo " Pulling elasticsearch ... done"
@echo " Pulling discovery ... done"
@echo " Pulling forum ... done"
@echo " Pulling devpi ... done"
@echo " Pulling lms ... done"
@echo "~/devstack$$"
@echo -n $(YELLOW)
@echo "----------------------------------"
@echo "If you must pull all images, such as for initial"
@echo "provisioning, run 'make dev.pull'."
@echo -n $(RED)
@echo "****************************************************************"
@echo -n $(NO_COLOR)
pull.xqueue: dev.pull.without-deps.xqueue+xqueue_consumer
validate: ## Validate the devstack configuration
docker-compose $(DOCKER_COMPOSE_FILES) config
backup: ## Write all data volumes to the host.
docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql
docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data
restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA!
docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz
docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
# TODO: Print out help for this target. Even better if we can iterate over the
# services in docker-compose.yml, and print the actual service names.
%-shell: ## Run a shell on the specified service container
docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash
analyticspipeline-shell: ## Run a shell on the analytics pipeline container
docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open
credentials-shell: ## Run a shell on the credentials container
docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
discovery-shell: ## Run a shell on the discovery container
docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open
ecommerce-shell: ## Run a shell on the ecommerce container
docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open
e2e-shell: ## Start the end-to-end tests container with a shell
docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash
registrar-shell: ## Run a shell on the registrar site container
docker-compose $(DOCKER_COMPOSE_FILES) exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open
%-update-db: ## Run migrations for the specified service container
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'
studio-update-db: ## Run migrations for the Studio container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'
lms-update-db: ## Run migrations LMS container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'
update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES
forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it.
docker-compose $(DOCKER_COMPOSE_FILES) forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'
lms-shell: ## Run a shell on the LMS container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
lms-watcher-shell: ## Run a shell on the LMS watcher container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
%-attach: ## Attach to the specified service container process to use the debugger & see logs.
docker attach "$$(make --silent dev.print-container.$*)"
lms-restart: lms-restart-devserver
studio-shell: ## Run a shell on the Studio container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
studio-watcher-shell: ## Run a shell on the studio watcher container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
studio-restart: studio-restart-devserver
xqueue-shell: ## Run a shell on the XQueue container
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
xqueue-restart: xqueue-restart-devserver
%-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it.
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'
xqueue_consumer-shell: ## Run a shell on the XQueue consumer container
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it.
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")'
%-static: ## Rebuild static assets for the specified service container
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
lms-sass: ## Rebuild sass assets for the LMS container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/var/edxapp/themes/'
lms-static: ## Rebuild static assets for the LMS container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/var/edxapp/themes/'
studio-sass: ## Rebuild sass assets for the Studio container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/var/edxapp/themes/'
studio-static: ## Rebuild static assets for the Studio container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/var/edxapp/themes/'
## Appsembler: Remove `credentials-static`, `discovery-static` and `ecommerce-static`
## because they're problematic and not needed for us
credentials-static:
@echo 'credentials-static: Disabled at Appsembler'
discovery-static:
@echo 'discovery-static: Disabled at Appsembler'
ecommerce-static:
@echo 'ecommerce-static: Disabled at Appsembler'
static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers
healthchecks: dev.check.$(DEFAULT_SERVICES)
healthchecks.%: dev.check.%
e2e-tests: ## Run the end-to-end tests against the service containers
docker run -t --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"'
validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container
touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile
docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile
rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile
vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers
@docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1
@docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1
devpi-password: ## Get the root devpi password for the devpi container
docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword"
mysql-shell: ## Run a shell on the mysql container
docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash
mysql-shell-edxapp: ## Run a mysql shell on the edxapp database
docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp"
mongo-shell: ## Run a shell on the mongo container
docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash
dev.provision.analytics_pipeline: dev.provision.services.analyticspipeline
analytics-pipeline-shell: analyticspipeline-shell
dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeline services
pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images
analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build
docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity'
stop.analytics_pipeline: ## Stop all Analytics pipeline services.
docker-compose $(DOCKER_COMPOSE_FILES) stop \
namenode datanode resourcemanager nodemanager sparkmaster \
sparkworker vertica analyticspipeline
hadoop-application-logs-%: ## View hadoop logs by application Id
docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $*
# Provisions studio, ecommerce, and marketing with course(s) in test-course.json
# Modify test-course.json before running this make target to generate a custom course
create-test-course: ## NOTE: marketing course creation is not available for those outside edX
$(WINPTY) bash ./course-generator/create-courses.sh --studio --ecommerce --marketing course-generator/test-course.json
# Run the course json builder script and use the outputted course json to provision studio, ecommerce, and marketing
# Modify the list of courses in build-course-json.sh beforehand to generate custom courses
build-courses: ## NOTE: marketing course creation is not available for those outside edX
$(WINPTY) bash ./course-generator/build-course-json.sh course-generator/tmp-config.json
$(WINPTY) bash ./course-generator/create-courses.sh --studio --ecommerce --marketing course-generator/tmp-config.json
rm course-generator/tmp-config.json
check-memory: ## Check if enough memory has been allocated to Docker
@if [ `docker info --format '{{.MemTotal}}'` -lt 2095771648 ]; then echo "\033[0;31mWarning, System Memory is set too low!!! Increase Docker memory to be at least 2 Gigs\033[0m"; fi || exit 0
stats: ## Get per-container CPU and memory utilization data
docker stats --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
feature-toggle-state: ## Gather the state of feature toggles configured for various IDAs
$(WINPTY) bash ./gather-feature-toggle-state.sh
selfcheck: ## check that the Makefile is well-formed
@echo "The Makefile is well-formed."