forked from WikiCommunityHealth/wikidump
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
153 lines (129 loc) · 5.6 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
# FOLDERS
VENV := venv
PROJECT_NAME := wikidump
DUMP_FOLDER := dumps
CAWIKI := cawiki
ITWIKI := itwiki
DEWIKI := dewiki
ENWIKI := enwiki
ESWIKI := eswiki
FRWIKI := frwiki
JAWIKI := jawiki
NLWIKI := nlwiki
PLWIKI := plwiki
PTWIKI := ptwiki
RUWIKI := ruwiki
SVWIKI := svwiki
ZHWIKI := zhwiki
# PROGRAMS AND FLAGS
PYFLAGS := -m
PROGRAM := wikidump
OUTPUT_FOLDER := output_user_warnings_probabilistic
PROGRAM_FLAGS := --output-compression gzip
FUNCTION_TO_RUN := extract-user-warnings-templates-probabilistic
FUNCTION_SUB_COMMANDS := --only-pages-with-user-warnings --language catalan output_tokens/cawiki-20210201-pages-meta-history.xml.7z.features.json.gz --only-last-revision
PYTHON := python3
PIP := pip
DUMP_EXT := .7z
# COLORS
RED := \033[31m
GREEN := \033[32m
YELLOW := \033[33m
BLUE := \033[34m
NONE := \033[0m
# COMMANDS
ECHO := echo -e
MKDIR := mkdir -p
RM := rm -rf
# RULES
.PHONY: help env deleteenv install install-dev run-ca run-it run-all run-de run-es run-fr run-ja run-nl run-pl run-pt run-ru run-sv run-zh
help:
@$(ECHO) '$(YELLOW)Makefile help$(NONE)'
env:
@$(ECHO) '$(GREEN)Creating the virtual environment..$(NONE)'
@$(MKDIR) $(VENV)
@$(eval PYTHON_VERSION=$(shell $(PYTHON) --version | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]' | cut -f1,2 -d'.'))
@$(PYTHON_VERSION) -m venv $(VENV)/$(PROJECT_NAME)
@$(ECHO) '$(GREEN)Done$(NONE)'
install:
@$(ECHO) '$(GREEN)Installing requirements..$(NONE)'
@pip install -r requirements.txt
@$(ECHO) '$(GREEN)Done$(NONE)'
install-dev:
@$(ECHO) '$(GREEN)Installing requirements..$(NONE)'
@$(PIP) install -r requirements.dev.txt
@$(ECHO) '$(GREEN)Done$(NONE)'
run-ca:
@$(ECHO) '$(BLUE)Running cawiki datasets..$(NONE)'
@$(eval FILES=$(wildcard $(DUMP_FOLDER)/$(CAWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-it:
@$(ECHO) '$(BLUE)Running itwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(ITWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-de:
@$(ECHO) '$(BLUE)Running dewiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(DEWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-es:
@$(ECHO) '$(BLUE)Running eswiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(ESWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-en:
@$(ECHO) '$(BLUE)Running enwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(ENWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-fr:
@$(ECHO) '$(BLUE)Running frwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(FRWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-ja:
@$(ECHO) '$(BLUE)Running jawiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(JAWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-nl:
@$(ECHO) '$(BLUE)Running nlwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(NLWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-pl:
@$(ECHO) '$(BLUE)Running plwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(PLWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-pt:
@$(ECHO) '$(BLUE)Running ptwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(PTWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-ru:
@$(ECHO) '$(BLUE)Running ruwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(RUWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-sv:
@$(ECHO) '$(BLUE)Running svwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(SVWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-zh:
@$(ECHO) '$(BLUE)Running zhwiki datasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/$(ZHWIKI)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
run-all:
@$(ECHO) '$(BLUE)Running all the wikidatasets..$(NONE)'
@$(eval FILES=$(shell find $(DUMP_FOLDER)/**/*$(DUMP_EXT)))
@$(PYTHON) $(PYFLAGS) $(PROGRAM) $(PROGRAM_FLAGS) $(FILES) $(OUTPUT_FOLDER) $(FUNCTION_TO_RUN) $(FUNCTION_SUB_COMMANDS)
@$(ECHO) '$(BLUE)Done$(NONE)'
deleteenv:
@$(ECHO) '$(RED)Deleting the virtual environment..$(NONE)'
@$(RM) $(VENV)
@$(ECHO) '$(RED)Done$(NONE)'