diff --git a/crawlab/.gitignore b/crawlab/.gitignore index e043754d7..ccc818410 100644 --- a/crawlab/.gitignore +++ b/crawlab/.gitignore @@ -110,3 +110,5 @@ node_modules/ # egg-info *.egg-info + +tmp/ diff --git a/crawlab/app.py b/crawlab/app.py index c984f1405..3a60c19e7 100644 --- a/crawlab/app.py +++ b/crawlab/app.py @@ -71,7 +71,8 @@ '/api/schedules/') api.add_resource(SiteApi, '/api/sites', - '/api/sites/') + '/api/sites/', + '/api/sites/get/') def monitor_nodes_status(celery_app): diff --git a/crawlab/bin/run_worker.py b/crawlab/bin/run_worker.py index 92bf85d5f..69edcbc68 100644 --- a/crawlab/bin/run_worker.py +++ b/crawlab/bin/run_worker.py @@ -13,7 +13,7 @@ import tasks.deploy if __name__ == '__main__': - if sys.platform == 'windows': + if 'win' in sys.platform: celery_app.start(argv=['tasks', 'worker', '-P', 'eventlet', '-E', '-l', 'INFO']) else: celery_app.start(argv=['tasks', 'worker', '-E', '-l', 'INFO']) diff --git a/crawlab/constants/spider.py b/crawlab/constants/spider.py index 685e2b072..97cbbdf2e 100644 --- a/crawlab/constants/spider.py +++ b/crawlab/constants/spider.py @@ -1,7 +1,6 @@ class SpiderType: - SCRAPY = 'scrapy' - PYSPIDER = 'pyspider' - WEBMAGIC = 'webmagic' + CONFIGURABLE = 'configurable' + CUSTOMIZED = 'customized' class LangType: @@ -17,6 +16,22 @@ class CronEnabled: OFF = 0 +class CrawlType: + LIST = 'list' + DETAIL = 'detail' + LIST_DETAIL = 'list-detail' + + +class QueryType: + CSS = 'css' + XPATH = 'xpath' + + +class ExtractType: + TEXT = 'text' + ATTRIBUTE = 'attribute' + + SUFFIX_IGNORE = [ 'pyc' ] diff --git a/crawlab/db/manager.py b/crawlab/db/manager.py index 4c5535e70..e80aada66 100644 --- a/crawlab/db/manager.py +++ b/crawlab/db/manager.py @@ -179,5 +179,9 @@ def create_index(self, col_name: str, keys: dict, **kwargs): col = self.db[col_name] col.create_index(keys=keys, **kwargs) + def distinct(self, col_name: str, key: str, filter: dict): + col = self.db[col_name] + return sorted(col.distinct(key, filter)) + db_manager = DbManager() diff --git a/crawlab/requirements.txt b/crawlab/requirements.txt index a54862f29..11af6f698 100644 --- a/crawlab/requirements.txt +++ b/crawlab/requirements.txt @@ -1,35 +1,79 @@ +aiohttp==3.5.4 amqp==2.4.2 aniso8601==6.0.0 +Appium-Python-Client==0.40 APScheduler==3.6.0 +asn1crypto==0.24.0 +async-timeout==3.0.1 attrs==19.1.0 +Automat==0.7.0 Babel==2.6.0 +beautifulsoup4==4.7.1 billiard==3.6.0.0 +bs4==0.0.1 +cachetools==3.1.0 celery==4.3.0 certifi==2019.3.9 +cffi==1.12.3 chardet==3.0.4 Click==7.0 coloredlogs==10.0 +constantly==15.1.0 +cryptography==2.6.1 +cssselect==1.0.3 Flask==1.0.2 +Flask-APScheduler==1.11.0 Flask-Cors==3.0.7 Flask-RESTful==0.3.7 flask-restplus==0.12.1 flower==0.9.3 +gevent==1.4.0 +greenlet==0.4.15 +gunicorn==19.9.0 +html5lib==1.0.1 humanfriendly==4.18 +hyperlink==19.0.0 idna==2.8 +idna-ssl==1.1.0 +incremental==17.5.0 itsdangerous==1.1.0 Jinja2==2.10 +jsonpickle==1.1 jsonschema==3.0.1 kombu==4.5.0 +lxml==4.3.3 MarkupSafe==1.1.1 mongoengine==0.17.0 +multidict==4.5.2 +parsel==1.5.1 +pyasn1==0.4.5 +pyasn1-modules==0.2.5 +pycparser==2.19 +PyDispatcher==2.0.5 +PyHamcrest==1.9.0 pymongo==3.7.2 +pyOpenSSL==19.0.0 pyrsistent==0.14.11 +python-dateutil==2.8.0 pytz==2018.9 +queuelib==1.5.0 redis==3.2.1 +redisbeat==1.1.4 +reppy==0.4.12 requests==2.21.0 +Scrapy==1.6.0 +selenium==3.141.0 +service-identity==18.1.0 six==1.12.0 +soupsieve==1.9.1 tornado==5.1.1 +Twisted==19.2.0 +typing-extensions==3.7.2 tzlocal==1.5.1 urllib3==1.24.1 vine==1.3.0 +w3lib==1.20.0 +webencodings==0.5.1 Werkzeug==0.15.2 +yarl==1.3.0 +zope.interface==4.6.0 diff --git a/crawlab/routes/base.py b/crawlab/routes/base.py index 1578b3f84..e068e4a23 100644 --- a/crawlab/routes/base.py +++ b/crawlab/routes/base.py @@ -111,7 +111,7 @@ def put(self) -> (dict, tuple): self.after_update() - return item + return jsonify(item) def update(self, id: str = None) -> (dict, tuple): """ @@ -137,7 +137,7 @@ def update(self, id: str = None) -> (dict, tuple): # execute after_update hook self.after_update(id) - return item + return jsonify(item) def post(self, id: str = None, action: str = None): """ diff --git a/crawlab/routes/sites.py b/crawlab/routes/sites.py index 6874af2b9..e49dbe37f 100644 --- a/crawlab/routes/sites.py +++ b/crawlab/routes/sites.py @@ -13,6 +13,7 @@ class SiteApi(BaseApi): arguments = ( ('keyword', str), + ('main_category', str), ('category', str), ) @@ -70,3 +71,20 @@ def get(self, id: str = None, action: str = None): 'page_size': page_size, 'items': jsonify(sites) } + + def get_main_category_list(self, id): + return { + 'status': 'ok', + 'items': db_manager.distinct(col_name=self.col_name, key='main_category', filter={}) + } + + def get_category_list(self, id): + args = self.parser.parse_args() + filter_ = {} + if args.get('main_category') is not None: + filter_['main_category'] = args.get('main_category') + return { + 'status': 'ok', + 'items': db_manager.distinct(col_name=self.col_name, key='category', + filter=filter_) + } diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index 51bae78ca..243f11a13 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -5,23 +5,27 @@ from datetime import datetime from random import random +import gevent import requests from bson import ObjectId from flask import current_app, request from flask_restful import reqparse, Resource +from lxml import etree from werkzeug.datastructures import FileStorage from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_SOURCE_FILE_FOLDER, PROJECT_TMP_FOLDER from constants.node import NodeStatus +from constants.spider import SpiderType, CrawlType, QueryType, ExtractType from constants.task import TaskStatus from db.manager import db_manager from routes.base import BaseApi from tasks.scheduler import scheduler -from tasks.spider import execute_spider +from tasks.spider import execute_spider, execute_config_spider from utils import jsonify from utils.deploy import zip_file, unzip_file from utils.file import get_file_suffix_stats, get_file_suffix -from utils.spider import get_lang_by_stats, get_last_n_run_errors_count, get_last_n_day_tasks_count +from utils.spider import get_lang_by_stats, get_last_n_run_errors_count, get_last_n_day_tasks_count, get_list_page_data, \ + get_detail_page_data parser = reqparse.RequestParser() parser.add_argument('file', type=FileStorage, location='files') @@ -64,6 +68,37 @@ class SpiderApi(BaseApi): # spider site ('site', str), + + ######################## + # Configurable Spider + ######################## + + # spider crawl fields for list page + ('fields', str), + + # spider crawl fields for detail page + ('detail_fields', str), + + # spider crawl type + ('crawl_type', str), + + # spider start url + ('start_url', str), + + # spider item selector + ('item_selector', str), + + # spider item selector type + ('item_selector_type', str), + + # spider pagination selector + ('pagination_selector', str), + + # spider pagination selector type + ('pagination_selector_type', str), + + # whether to obey robots.txt + ('obey_robots_txt', str), ) def get(self, id=None, action=None): @@ -96,6 +131,8 @@ def get(self, id=None, action=None): # get a list of items else: items = [] + + # get customized spiders dirs = os.listdir(PROJECT_SOURCE_FILE_FOLDER) for _dir in dirs: if _dir in IGNORE_DIRS: @@ -114,6 +151,7 @@ def get(self, id=None, action=None): 'src': dir_path, 'lang': lang, 'suffix_stats': stats, + 'type': SpiderType.CUSTOMIZED }) # existing spider @@ -123,39 +161,52 @@ def get(self, id=None, action=None): if last_deploy is not None: spider['deploy_ts'] = last_deploy['finish_ts'] - # get last task - last_task = db_manager.get_last_task(spider_id=spider['_id']) - if last_task is not None: - spider['task_ts'] = last_task['create_ts'] - - # get site - if spider.get('site') is not None: - site = db_manager.get('sites', spider['site']) - if site is not None: - spider['site_name'] = site['name'] - # file stats stats = get_file_suffix_stats(dir_path) # language lang = get_lang_by_stats(stats) + # spider type + type_ = SpiderType.CUSTOMIZED + # update spider data db_manager.update_one('spiders', id=str(spider['_id']), values={ 'lang': lang, + 'type': type_, 'suffix_stats': stats, }) - # --------- - # stats - # --------- - # last 5-run errors - spider['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5) - spider['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5) + # append spider + items.append(spider) + # get configurable spiders + for spider in db_manager.list('spiders', {'type': SpiderType.CONFIGURABLE}): # append spider items.append(spider) + # get other info + for i in range(len(items)): + spider = items[i] + + # get site + if spider.get('site') is not None: + site = db_manager.get('sites', spider['site']) + if site is not None: + items[i]['site_name'] = site['name'] + + # get last task + last_task = db_manager.get_last_task(spider_id=spider['_id']) + if last_task is not None: + items[i]['task_ts'] = last_task['create_ts'] + + # --------- + # stats + # --------- + # last 5-run errors + items[i]['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5) + items[i]['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5) + return { 'status': 'ok', 'items': jsonify(items) @@ -214,7 +265,16 @@ def on_crawl(self, id: str) -> (dict, tuple): spider = db_manager.get('spiders', id=ObjectId(id)) - job = execute_spider.delay(id, params) + # determine execute function + if spider['type'] == SpiderType.CONFIGURABLE: + # configurable spider + exec_func = execute_config_spider + else: + # customized spider + exec_func = execute_spider + + # trigger an asynchronous job + job = exec_func.delay(id, params) # create a new task db_manager.save('tasks', { @@ -377,10 +437,101 @@ def after_update(self, id: str = None) -> None: scheduler.update() def update_envs(self, id: str): + """ + Update environment variables + :param id: spider_id + """ args = self.parser.parse_args() envs = json.loads(args.envs) db_manager.update_one(col_name='spiders', id=id, values={'envs': envs}) + def update_fields(self, id: str): + """ + Update list page fields variables for configurable spiders + :param id: spider_id + """ + args = self.parser.parse_args() + fields = json.loads(args.fields) + db_manager.update_one(col_name='spiders', id=id, values={'fields': fields}) + + def update_detail_fields(self, id: str): + """ + Update detail page fields variables for configurable spiders + :param id: spider_id + """ + args = self.parser.parse_args() + detail_fields = json.loads(args.detail_fields) + db_manager.update_one(col_name='spiders', id=id, values={'detail_fields': detail_fields}) + + def preview_crawl(self, id: str): + spider = db_manager.get(col_name='spiders', id=id) + + if spider['type'] != SpiderType.CONFIGURABLE: + return { + 'status': 'ok', + 'error': 'type %s is invalid' % spider['type'] + }, 400 + + if spider.get('start_url') is None: + return { + 'status': 'ok', + 'error': 'start_url should not be empty' + }, 400 + + try: + r = requests.get(spider['start_url']) + except Exception as err: + return { + 'status': 'ok', + 'error': 'connection error' + }, 500 + + if r.status_code != 200: + return { + 'status': 'ok', + 'error': 'status code is not 200, but %s' % r.status_code + } + + # get html parse tree + sel = etree.HTML(r.content) + + # parse fields + if spider['crawl_type'] == CrawlType.LIST: + if spider.get('item_selector') is None: + return { + 'status': 'ok', + 'error': 'item_selector should not be empty' + }, 400 + + data = get_list_page_data(spider, sel)[:10] + + return { + 'status': 'ok', + 'items': data + } + + elif spider['crawl_type'] == CrawlType.DETAIL: + pass + + elif spider['crawl_type'] == CrawlType.LIST_DETAIL: + data = get_list_page_data(spider, sel)[:10] + + ev_list = [] + for idx, d in enumerate(data): + for f in spider['fields']: + if f.get('is_detail'): + url = d.get(f['name']) + if url is not None: + ev_list.append(gevent.spawn(get_detail_page_data, url, spider, idx, data)) + break + + gevent.joinall(ev_list) + + return { + 'status': 'ok', + 'items': data + } + class SpiderImportApi(Resource): __doc__ = """ diff --git a/crawlab/spiders/scrapy.cfg b/crawlab/spiders/scrapy.cfg new file mode 100644 index 000000000..bf9391f15 --- /dev/null +++ b/crawlab/spiders/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = spiders.settings + +[deploy] +#url = http://localhost:6800/ +project = spiders diff --git a/crawlab/spiders/spiders/__init__.py b/crawlab/spiders/spiders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/crawlab/spiders/spiders/db.py b/crawlab/spiders/spiders/db.py new file mode 100644 index 000000000..18925f8d9 --- /dev/null +++ b/crawlab/spiders/spiders/db.py @@ -0,0 +1,14 @@ +import os + +from pymongo import MongoClient + +MONGO_HOST = os.environ.get('MONGO_HOST') +MONGO_PORT = int(os.environ.get('MONGO_PORT')) +MONGO_DB = os.environ.get('MONGO_DB') +mongo = MongoClient(host=MONGO_HOST, + port=MONGO_PORT) +db = mongo[MONGO_DB] +task_id = os.environ.get('CRAWLAB_TASK_ID') +col_name = os.environ.get('CRAWLAB_COLLECTION') +task = db['tasks'].find_one({'_id': task_id}) +spider = db['spiders'].find_one({'_id': task['spider_id']}) diff --git a/crawlab/spiders/spiders/items.py b/crawlab/spiders/spiders/items.py new file mode 100644 index 000000000..7163d6e3f --- /dev/null +++ b/crawlab/spiders/spiders/items.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/items.html + +import scrapy + +from spiders.db import spider + + +class SpidersItem(scrapy.Item): + if spider['crawl_type'] == 'list': + fields = {f['name']: scrapy.Field() for f in spider['fields']} + elif spider['crawl_type'] == 'detail': + fields = {f['name']: scrapy.Field() for f in spider['detail_fields']} + elif spider['crawl_type'] == 'list-detail': + fields = {f['name']: scrapy.Field() for f in (spider['fields'] + spider['detail_fields'])} + else: + fields = {} + + # basic fields + fields['_id'] = scrapy.Field() + fields['task_id'] = scrapy.Field() diff --git a/crawlab/spiders/spiders/middlewares.py b/crawlab/spiders/spiders/middlewares.py new file mode 100644 index 000000000..1760fe411 --- /dev/null +++ b/crawlab/spiders/spiders/middlewares.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class SpidersSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class SpidersDownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/crawlab/spiders/spiders/pipelines.py b/crawlab/spiders/spiders/pipelines.py new file mode 100644 index 000000000..695310675 --- /dev/null +++ b/crawlab/spiders/spiders/pipelines.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +from spiders.db import db, col_name, task_id + + +class SpidersPipeline(object): + col = db[col_name] + + def process_item(self, item, spider): + item['task_id'] = task_id + self.col.save(item) + + return item diff --git a/crawlab/spiders/spiders/settings.py b/crawlab/spiders/spiders/settings.py new file mode 100644 index 000000000..d78d9281e --- /dev/null +++ b/crawlab/spiders/spiders/settings.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for spiders project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +from spiders.db import spider + +BOT_NAME = 'Crawlab Spider' + +SPIDER_MODULES = ['spiders.spiders'] +NEWSPIDER_MODULE = 'spiders.spiders' + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +# USER_AGENT = 'spiders (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +# CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +# DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +# CONCURRENT_REQUESTS_PER_DOMAIN = 16 +# CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +# COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +# TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +# DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +# } + +# Enable or disable spider middlewares +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# SPIDER_MIDDLEWARES = { +# 'spiders.middlewares.SpidersSpiderMiddleware': 543, +# } + +# Enable or disable downloader middlewares +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# DOWNLOADER_MIDDLEWARES = { +# 'spiders.middlewares.SpidersDownloaderMiddleware': 543, +# } + +# Enable or disable extensions +# See https://doc.scrapy.org/en/latest/topics/extensions.html +# EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +# } + +# Configure item pipelines +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'spiders.pipelines.SpidersPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +# AUTOTHROTTLE_ENABLED = True +# The initial download delay +# AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +# AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +# AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# HTTPCACHE_ENABLED = True +# HTTPCACHE_EXPIRATION_SECS = 0 +# HTTPCACHE_DIR = 'httpcache' +# HTTPCACHE_IGNORE_HTTP_CODES = [] +# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/crawlab/spiders/spiders/spiders/__init__.py b/crawlab/spiders/spiders/spiders/__init__.py new file mode 100644 index 000000000..ebd689ac5 --- /dev/null +++ b/crawlab/spiders/spiders/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/crawlab/spiders/spiders/spiders/config_spider.py b/crawlab/spiders/spiders/spiders/config_spider.py new file mode 100644 index 000000000..fe801f8ef --- /dev/null +++ b/crawlab/spiders/spiders/spiders/config_spider.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +from urllib.parse import urlparse + +import scrapy + +from spiders.db import spider +from spiders.items import SpidersItem + + +def get_detail_url(item): + for f in spider['fields']: + if f.get('is_detail'): + return item.get(f['name']) + return None + + +def get_spiders_item(sel, fields, item=None): + if item is None: + item = SpidersItem() + + for f in fields: + if f['type'] == 'xpath': + # xpath selector + if f['extract_type'] == 'text': + # text content + query = f['query'] + '/text()' + else: + # attribute + attribute = f["attribute"] + query = f['query'] + f'/@("{attribute}")' + item[f['name']] = sel.xpath(query).extract_first() + + else: + # css selector + if f['extract_type'] == 'text': + # text content + query = f['query'] + '::text' + else: + # attribute + attribute = f["attribute"] + query = f['query'] + f'::attr("{attribute}")' + item[f['name']] = sel.css(query).extract_first() + + return item + + +def get_list_items(response): + if spider['item_selector_type'] == 'xpath': + # xpath selector + items = response.xpath(spider['item_selector']) + else: + # css selector + items = response.css(spider['item_selector']) + return items + + +def get_next_url(response): + # pagination + if spider.get('pagination_selector') is not None: + if spider['pagination_selector_type'] == 'xpath': + # xpath selector + next_url = response.xpath(spider['pagination_selector'] + '/@href').extract_first() + else: + # css selector + next_url = response.css(spider['pagination_selector'] + '::attr("href")').extract_first() + + # found next url + if next_url is not None: + if not next_url.startswith('http') and not next_url.startswith('//'): + u = urlparse(response.url) + next_url = f'{u.scheme}://{u.netloc}{next_url}' + return next_url + return None + + +class ConfigSpiderSpider(scrapy.Spider): + name = 'config_spider' + # allowed_domains = [] + start_urls = [spider['start_url']] + + def parse(self, response): + + if spider['crawl_type'] == 'list': + # list page only + items = get_list_items(response) + for _item in items: + item = get_spiders_item(sel=_item, fields=spider['fields']) + yield item + next_url = get_next_url(response) + if next_url is not None: + yield scrapy.Request(url=next_url) + + elif spider['crawl_type'] == 'detail': + # TODO: detail page onlny + # detail page only + pass + + elif spider['crawl_type'] == 'list-detail': + # list page + detail page + items = get_list_items(response) + for _item in items: + item = get_spiders_item(sel=_item, fields=spider['fields']) + detail_url = get_detail_url(item) + if detail_url is not None: + yield scrapy.Request(url=detail_url, + callback=self.parse_detail, + meta={ + 'item': item + }) + next_url = get_next_url(response) + if next_url is not None: + yield scrapy.Request(url=next_url) + + def parse_detail(self, response): + item = get_spiders_item(sel=response, fields=spider['detail_fields'], item=response.meta['item']) + yield item diff --git a/crawlab/tasks/spider.py b/crawlab/tasks/spider.py index 48cafc273..3bdc65bcd 100644 --- a/crawlab/tasks/spider.py +++ b/crawlab/tasks/spider.py @@ -6,13 +6,15 @@ from bson import ObjectId from pymongo import ASCENDING, DESCENDING -from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH +from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH, MONGO_HOST, MONGO_PORT, MONGO_DB from constants.task import TaskStatus from db.manager import db_manager from .celery import celery_app import subprocess from utils.log import other as logger +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + def get_task(id: str): i = 0 @@ -112,6 +114,125 @@ def execute_spider(self, id: str, params: str = None): env=env, bufsize=1) + # update pid + db_manager.update_one(col_name='tasks', id=task_id, values={ + 'pid': p.pid + }) + + # get output from the process + _stdout, _stderr = p.communicate() + + # get return code + code = p.poll() + if code == 0: + status = TaskStatus.SUCCESS + else: + status = TaskStatus.FAILURE + except Exception as err: + logger.error(err) + stderr.write(str(err)) + status = TaskStatus.FAILURE + + # save task when the task is finished + finish_ts = datetime.utcnow() + db_manager.update_one('tasks', id=task_id, values={ + 'finish_ts': finish_ts, + 'duration': (finish_ts - task['create_ts']).total_seconds(), + 'status': status + }) + task = db_manager.get('tasks', id=id) + + # close log file streams + stdout.flush() + stderr.flush() + stdout.close() + stderr.close() + + return task + + +@celery_app.task(bind=True) +def execute_config_spider(self, id: str, params: str = None): + task_id = self.request.id + hostname = self.request.hostname + spider = db_manager.get('spiders', id=id) + + # get task object and return if not found + task = get_task(task_id) + if task is None: + return + + # current working directory + current_working_directory = os.path.join(BASE_DIR, 'spiders') + + # log info + logger.info('task_id: %s' % task_id) + logger.info('hostname: %s' % hostname) + logger.info('current_working_directory: %s' % current_working_directory) + logger.info('spider_id: %s' % id) + + # make sure the log folder exists + log_path = os.path.join(PROJECT_LOGS_FOLDER, id) + if not os.path.exists(log_path): + os.makedirs(log_path) + + # open log file streams + log_file_path = os.path.join(log_path, '%s.log' % datetime.now().strftime('%Y%m%d%H%M%S')) + stdout = open(log_file_path, 'a') + stderr = open(log_file_path, 'a') + + # update task status as started + db_manager.update_one('tasks', id=task_id, values={ + 'start_ts': datetime.utcnow(), + 'node_id': hostname, + 'hostname': hostname, + 'log_file_path': log_file_path, + 'status': TaskStatus.STARTED + }) + + # pass params as env variables + env = os.environ.copy() + + # custom environment variables + if spider.get('envs'): + for _env in spider.get('envs'): + env[_env['name']] = _env['value'] + + # task id environment variable + env['CRAWLAB_TASK_ID'] = task_id + + # collection environment variable + if spider.get('col'): + env['CRAWLAB_COLLECTION'] = spider.get('col') + + # create index to speed results data retrieval + db_manager.create_index(spider.get('col'), [('task_id', ASCENDING)]) + + # mongodb environment variables + env['MONGO_HOST'] = MONGO_HOST + env['MONGO_PORT'] = str(MONGO_PORT) + env['MONGO_DB'] = MONGO_DB + + cmd_arr = [ + sys.executable, + '-m', + 'scrapy', + 'crawl', + 'config_spider' + ] + try: + p = subprocess.Popen(cmd_arr, + stdout=stdout.fileno(), + stderr=stderr.fileno(), + cwd=current_working_directory, + env=env, + bufsize=1) + + # update pid + db_manager.update_one(col_name='tasks', id=task_id, values={ + 'pid': p.pid + }) + # get output from the process _stdout, _stderr = p.communicate() diff --git a/crawlab/utils/spider.py b/crawlab/utils/spider.py index 6f7d4ef67..8720c50f3 100644 --- a/crawlab/utils/spider.py +++ b/crawlab/utils/spider.py @@ -1,9 +1,11 @@ import os +import requests from datetime import datetime, timedelta from bson import ObjectId +from lxml import etree -from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType +from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType, QueryType, ExtractType from constants.task import TaskStatus from db.manager import db_manager @@ -69,3 +71,53 @@ def get_last_n_day_tasks_count(spider_id: ObjectId, n: int) -> list: '$gte': (datetime.now() - timedelta(n)) } }) + + +def get_list_page_data(spider, sel): + data = [] + if spider['item_selector_type'] == QueryType.XPATH: + items = sel.xpath(spider['item_selector']) + else: + items = sel.cssselect(spider['item_selector']) + for item in items: + row = {} + for f in spider['fields']: + if f['type'] == QueryType.CSS: + # css selector + res = item.cssselect(f['query']) + else: + # xpath + res = item.xpath(f['query']) + + if len(res) > 0: + if f['extract_type'] == ExtractType.TEXT: + row[f['name']] = res[0].text + else: + row[f['name']] = res[0].get(f['attribute']) + data.append(row) + return data + + +def get_detail_page_data(url, spider, idx, data): + r = requests.get(url) + + sel = etree.HTML(r.content) + + row = {} + for f in spider['detail_fields']: + if f['type'] == QueryType.CSS: + # css selector + res = sel.cssselect(f['query']) + else: + # xpath + res = sel.xpath(f['query']) + + if len(res) > 0: + if f['extract_type'] == ExtractType.TEXT: + row[f['name']] = res[0].text + else: + row[f['name']] = res[0].get(f['attribute']) + + # assign values + for k, v in row.items(): + data[idx][k] = v diff --git a/docs/Architecture/App 2.md b/docs/Architecture/App 2.md deleted file mode 100644 index 5d5681fd8..000000000 --- a/docs/Architecture/App 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# App - diff --git a/docs/Concept/Deploy 2.md b/docs/Concept/Deploy 2.md deleted file mode 100644 index 12f55ebf6..000000000 --- a/docs/Concept/Deploy 2.md +++ /dev/null @@ -1,6 +0,0 @@ -# 部署 - -所有爬虫在运行前需要被部署当相应当节点中。 - -部署时，爬虫会被打包到相应的目录中，方便环境隔离，开发环境的爬虫和生产环境的爬虫需要打包部署来实现隔离。 - diff --git a/docs/Examples/README 2.md b/docs/Examples/README 2.md deleted file mode 100644 index 65afe6047..000000000 --- a/docs/Examples/README 2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Examples - diff --git a/docs/QuickStart/Installation 2.md b/docs/QuickStart/Installation 2.md deleted file mode 100644 index 3fce3e1c5..000000000 --- a/docs/QuickStart/Installation 2.md +++ /dev/null @@ -1,22 +0,0 @@ -# 安装 - -最快安装Crawlab的方式是克隆一份代码到本地 - -```bash -git clone https://github.com/tikazyq/crawlab -``` - -安装类库 - -```bash -# 安装后台类库 -pip install -r requirements.txt -``` - -```bash -# 安装前台类库 -cd frontend -npm install -``` - - diff --git a/docs/_book/Architecture/App 2.html b/docs/_book/Architecture/App 2.html deleted file mode 100644 index 0123e0e6f..000000000 --- a/docs/_book/Architecture/App 2.html +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - - App · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - - - - - -
- -
- -
- - - - - - - - -
-
- -
-
- -
- -

App

- - -
- -
-
-
- -

results matching ""

-
    - -
    -
    - -

    No results matching ""

    - -
    -
    -
    - -
    -
    - -
    - - - - - - - - - - - - - - -
    - - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/_book/Architecture/App.html b/docs/_book/Architecture/App.html index 0123e0e6f..9d7633048 100644 --- a/docs/_book/Architecture/App.html +++ b/docs/_book/Architecture/App.html @@ -397,7 +397,7 @@

    No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -407,6 +407,10 @@

    No results matching " + + + + diff --git a/docs/_book/Architecture/Celery.html b/docs/_book/Architecture/Celery.html index 128cf0232..38ef5b596 100644 --- a/docs/_book/Architecture/Celery.html +++ b/docs/_book/Architecture/Celery.html @@ -397,7 +397,7 @@

    No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -407,6 +407,10 @@

    No results matching " + + + + diff --git a/docs/_book/Architecture/index.html b/docs/_book/Architecture/index.html index 581f6f171..de925ce3f 100644 --- a/docs/_book/Architecture/index.html +++ b/docs/_book/Architecture/index.html @@ -397,7 +397,7 @@

    No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -407,6 +407,10 @@

    No results matching " + + + + diff --git a/docs/_book/Concept/Deploy 2.html b/docs/_book/Concept/Deploy 2.html deleted file mode 100644 index d7d110dbb..000000000 --- a/docs/_book/Concept/Deploy 2.html +++ /dev/null @@ -1,438 +0,0 @@ - - - - - - - 部署 · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - - - - - - - - -
    - -
    - -
    - - - - - - - - -
    -
    - -
    -
    - -
    - -

    部署

    -

    所有爬虫在运行前需要被部署当相应当节点中。

    -

    部署时，爬虫会被打包到相应的目录中，方便环境隔离，开发环境的爬虫和生产环境的爬虫需要打包部署来实现隔离。

    - - -
    - -
    -
    -
    - -

    results matching ""

    -
      - -
      -
      - -

      No results matching ""

      - -
      -
      -
      - -
      -
      - -
      - - - - - - - - - - - - - - -
      - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/_book/Concept/Deploy.html b/docs/_book/Concept/Deploy.html index d7d110dbb..1026cf5a3 100644 --- a/docs/_book/Concept/Deploy.html +++ b/docs/_book/Concept/Deploy.html @@ -399,7 +399,7 @@

      No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -409,6 +409,10 @@

      No results matching " + + + + diff --git a/docs/_book/Concept/Node.html b/docs/_book/Concept/Node.html index cb720eded..804289862 100644 --- a/docs/_book/Concept/Node.html +++ b/docs/_book/Concept/Node.html @@ -398,7 +398,7 @@

      No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -408,6 +408,10 @@

      No results matching " + + + + diff --git a/docs/_book/Concept/Spider.html b/docs/_book/Concept/Spider.html index 887c941e7..3021513da 100644 --- a/docs/_book/Concept/Spider.html +++ b/docs/_book/Concept/Spider.html @@ -403,7 +403,7 @@

      No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -413,6 +413,10 @@

      No results matching " + + + + diff --git a/docs/_book/Concept/Task.html b/docs/_book/Concept/Task.html index eca91bc30..6b2584cde 100644 --- a/docs/_book/Concept/Task.html +++ b/docs/_book/Concept/Task.html @@ -398,7 +398,7 @@

      No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -408,6 +408,10 @@

      No results matching " + + + + diff --git a/docs/_book/Concept/index.html b/docs/_book/Concept/index.html index b9d9296ab..9af874258 100644 --- a/docs/_book/Concept/index.html +++ b/docs/_book/Concept/index.html @@ -397,7 +397,7 @@

      No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -407,6 +407,10 @@

      No results matching " + + + + diff --git a/docs/_book/Examples/index 2.html b/docs/_book/Examples/index 2.html deleted file mode 100644 index 285296a55..000000000 --- a/docs/_book/Examples/index 2.html +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - - Examples · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      -
      - - - - - - - - -
      - -
      - -
      - - - - - - - - -
      -
      - -
      -
      - -
      - -

      Examples

      - - -
      - -
      -
      -
      - -

      results matching ""

      -
        - -
        -
        - -

        No results matching ""

        - -
        -
        -
        - -
        -
        - -
        - - - - - - - - - - - - - - -
        - - -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/_book/Examples/index.html b/docs/_book/Examples/index.html index 285296a55..f936aeaa9 100644 --- a/docs/_book/Examples/index.html +++ b/docs/_book/Examples/index.html @@ -397,7 +397,7 @@

        No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -407,6 +407,10 @@

        No results matching " + + + + diff --git a/docs/_book/QuickStart/Installation 2.html b/docs/_book/QuickStart/Installation 2.html deleted file mode 100644 index fcbd254fc..000000000 --- a/docs/_book/QuickStart/Installation 2.html +++ /dev/null @@ -1,447 +0,0 @@ - - - - - - - 安装 · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - -
        - -
        - -
        - - - - - - - - -
        -
        - -
        -
        - -
        - -

        安装

        -

        最快安装Crawlab的方式是克隆一份代码到本地

        -
        git clone https://github.com/tikazyq/crawlab
-
        -

        安装类库

        -
        # 安装后台类库
-pip install -r requirements.txt
-
        -
        # 安装前台类库
-cd frontend
-npm install
-
        - - -
        - -
        -
        -
        - -

        results matching ""

        -
          - -
          -
          - -

          No results matching ""

          - -
          -
          -
          - -
          -
          - -
          - - - - - - - - - - - - - - -
          - - -
          - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/_book/QuickStart/Installation.html b/docs/_book/QuickStart/Installation.html index fcbd254fc..49bad4b82 100644 --- a/docs/_book/QuickStart/Installation.html +++ b/docs/_book/QuickStart/Installation.html @@ -408,7 +408,7 @@

          No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -418,6 +418,10 @@

          No results matching " + + + + diff --git a/docs/_book/QuickStart/Run.html b/docs/_book/QuickStart/Run.html index c94c094b7..6018fc539 100644 --- a/docs/_book/QuickStart/Run.html +++ b/docs/_book/QuickStart/Run.html @@ -440,7 +440,7 @@

          No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -450,6 +450,10 @@

          No results matching " + + + + diff --git a/docs/_book/QuickStart/index.html b/docs/_book/QuickStart/index.html index c4cf78f87..9691dcadf 100644 --- a/docs/_book/QuickStart/index.html +++ b/docs/_book/QuickStart/index.html @@ -401,7 +401,7 @@

          No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}}); }); @@ -411,6 +411,10 @@

          No results matching " + + + + diff --git a/docs/_book/gitbook/fonts/fontawesome/FontAwesome 2.otf b/docs/_book/gitbook/fonts/fontawesome/FontAwesome 2.otf deleted file mode 100644 index d4de13e83..000000000 Binary files a/docs/_book/gitbook/fonts/fontawesome/FontAwesome 2.otf and /dev/null differ diff --git a/docs/_book/gitbook/gitbook-plugin-fontsettings/fontsettings 2.js b/docs/_book/gitbook/gitbook-plugin-fontsettings/fontsettings 2.js deleted file mode 100644 index ff7be7141..000000000 --- a/docs/_book/gitbook/gitbook-plugin-fontsettings/fontsettings 2.js +++ /dev/null @@ -1,240 +0,0 @@ -require(['gitbook', 'jquery'], function(gitbook, $) { - // Configuration - var MAX_SIZE = 4, - MIN_SIZE = 0, - BUTTON_ID; - - // Current fontsettings state - var fontState; - - // Default themes - var THEMES = [ - { - config: 'white', - text: 'White', - id: 0 - }, - { - config: 'sepia', - text: 'Sepia', - id: 1 - }, - { - config: 'night', - text: 'Night', - id: 2 - } - ]; - - // Default font families - var FAMILIES = [ - { - config: 'serif', - text: 'Serif', - id: 0 - }, - { - config: 'sans', - text: 'Sans', - id: 1 - } - ]; - - // Return configured themes - function getThemes() { - return THEMES; - } - - // Modify configured themes - function setThemes(themes) { - THEMES = themes; - updateButtons(); - } - - // Return configured font families - function getFamilies() { - return FAMILIES; - } - - // Modify configured font families - function setFamilies(families) { - FAMILIES = families; - updateButtons(); - } - - // Save current font settings - function saveFontSettings() { - gitbook.storage.set('fontState', fontState); - update(); - } - - // Increase font size - function enlargeFontSize(e) { - e.preventDefault(); - if (fontState.size >= MAX_SIZE) return; - - fontState.size++; - saveFontSettings(); - } - - // Decrease font size - function reduceFontSize(e) { - e.preventDefault(); - if (fontState.size <= MIN_SIZE) return; - - fontState.size--; - saveFontSettings(); - } - - // Change font family - function changeFontFamily(configName, e) { - if (e && e instanceof Event) { - e.preventDefault(); - } - - var familyId = getFontFamilyId(configName); - fontState.family = familyId; - saveFontSettings(); - } - - // Change type of color theme - function changeColorTheme(configName, e) { - if (e && e instanceof Event) { - e.preventDefault(); - } - - var $book = gitbook.state.$book; - - // Remove currently applied color theme - if (fontState.theme !== 0) - $book.removeClass('color-theme-'+fontState.theme); - - // Set new color theme - var themeId = getThemeId(configName); - fontState.theme = themeId; - if (fontState.theme !== 0) - $book.addClass('color-theme-'+fontState.theme); - - saveFontSettings(); - } - - // Return the correct id for a font-family config key - // Default to first font-family - function getFontFamilyId(configName) { - // Search for plugin configured font family - var configFamily = $.grep(FAMILIES, function(family) { - return family.config == configName; - })[0]; - // Fallback to default font family - return (!!configFamily)? configFamily.id : 0; - } - - // Return the correct id for a theme config key - // Default to first theme - function getThemeId(configName) { - // Search for plugin configured theme - var configTheme = $.grep(THEMES, function(theme) { - return theme.config == configName; - })[0]; - // Fallback to default theme - return (!!configTheme)? configTheme.id : 0; - } - - function update() { - var $book = gitbook.state.$book; - - $('.font-settings .font-family-list li').removeClass('active'); - $('.font-settings .font-family-list li:nth-child('+(fontState.family+1)+')').addClass('active'); - - $book[0].className = $book[0].className.replace(/\bfont-\S+/g, ''); - $book.addClass('font-size-'+fontState.size); - $book.addClass('font-family-'+fontState.family); - - if(fontState.theme !== 0) { - $book[0].className = $book[0].className.replace(/\bcolor-theme-\S+/g, ''); - $book.addClass('color-theme-'+fontState.theme); - } - } - - function init(config) { - // Search for plugin configured font family - var configFamily = getFontFamilyId(config.family), - configTheme = getThemeId(config.theme); - - // Instantiate font state object - fontState = gitbook.storage.get('fontState', { - size: config.size || 2, - family: configFamily, - theme: configTheme - }); - - update(); - } - - function updateButtons() { - // Remove existing fontsettings buttons - if (!!BUTTON_ID) { - gitbook.toolbar.removeButton(BUTTON_ID); - } - - // Create buttons in toolbar - BUTTON_ID = gitbook.toolbar.createButton({ - icon: 'fa fa-font', - label: 'Font Settings', - className: 'font-settings', - dropdown: [ - [ - { - text: 'A', - className: 'font-reduce', - onClick: reduceFontSize - }, - { - text: 'A', - className: 'font-enlarge', - onClick: enlargeFontSize - } - ], - $.map(FAMILIES, function(family) { - family.onClick = function(e) { - return changeFontFamily(family.config, e); - }; - - return family; - }), - $.map(THEMES, function(theme) { - theme.onClick = function(e) { - return changeColorTheme(theme.config, e); - }; - - return theme; - }) - ] - }); - } - - // Init configuration at start - gitbook.events.bind('start', function(e, config) { - var opts = config.fontsettings; - - // Generate buttons at start - updateButtons(); - - // Init current settings - init(opts); - }); - - // Expose API - gitbook.fontsettings = { - enlargeFontSize: enlargeFontSize, - reduceFontSize: reduceFontSize, - setTheme: changeColorTheme, - setFamily: changeFontFamily, - getThemes: getThemes, - setThemes: setThemes, - getFamilies: getFamilies, - setFamilies: setFamilies - }; -}); - - diff --git a/docs/_book/gitbook/gitbook-plugin-highlight/ebook 2.css b/docs/_book/gitbook/gitbook-plugin-highlight/ebook 2.css deleted file mode 100644 index cecaaab5a..000000000 --- a/docs/_book/gitbook/gitbook-plugin-highlight/ebook 2.css +++ /dev/null @@ -1,135 +0,0 @@ -pre, -code { - /* http://jmblog.github.io/color-themes-for-highlightjs */ - /* Tomorrow Comment */ - /* Tomorrow Red */ - /* Tomorrow Orange */ - /* Tomorrow Yellow */ - /* Tomorrow Green */ - /* Tomorrow Aqua */ - /* Tomorrow Blue */ - /* Tomorrow Purple */ -} -pre .hljs-comment, -code .hljs-comment, -pre .hljs-title, -code .hljs-title { - color: #8e908c; -} -pre .hljs-variable, -code .hljs-variable, -pre .hljs-attribute, -code .hljs-attribute, -pre .hljs-tag, -code .hljs-tag, -pre .hljs-regexp, -code .hljs-regexp, -pre .hljs-deletion, -code .hljs-deletion, -pre .ruby .hljs-constant, -code .ruby .hljs-constant, -pre .xml .hljs-tag .hljs-title, -code .xml .hljs-tag .hljs-title, -pre .xml .hljs-pi, -code .xml .hljs-pi, -pre .xml .hljs-doctype, -code .xml .hljs-doctype, -pre .html .hljs-doctype, -code .html .hljs-doctype, -pre .css .hljs-id, -code .css .hljs-id, -pre .css .hljs-class, -code .css .hljs-class, -pre .css .hljs-pseudo, -code .css .hljs-pseudo { - color: #c82829; -} -pre .hljs-number, -code .hljs-number, -pre .hljs-preprocessor, -code .hljs-preprocessor, -pre .hljs-pragma, -code .hljs-pragma, -pre .hljs-built_in, -code .hljs-built_in, -pre .hljs-literal, -code .hljs-literal, -pre .hljs-params, -code .hljs-params, -pre .hljs-constant, -code .hljs-constant { - color: #f5871f; -} -pre .ruby .hljs-class .hljs-title, -code .ruby .hljs-class .hljs-title, -pre .css .hljs-rules .hljs-attribute, -code .css .hljs-rules .hljs-attribute { - color: #eab700; -} -pre .hljs-string, -code .hljs-string, -pre .hljs-value, -code .hljs-value, -pre .hljs-inheritance, -code .hljs-inheritance, -pre .hljs-header, -code .hljs-header, -pre .hljs-addition, -code .hljs-addition, -pre .ruby .hljs-symbol, -code .ruby .hljs-symbol, -pre .xml .hljs-cdata, -code .xml .hljs-cdata { - color: #718c00; -} -pre .css .hljs-hexcolor, -code .css .hljs-hexcolor { - color: #3e999f; -} -pre .hljs-function, -code .hljs-function, -pre .python .hljs-decorator, -code .python .hljs-decorator, -pre .python .hljs-title, -code .python .hljs-title, -pre .ruby .hljs-function .hljs-title, -code .ruby .hljs-function .hljs-title, -pre .ruby .hljs-title .hljs-keyword, -code .ruby .hljs-title .hljs-keyword, -pre .perl .hljs-sub, -code .perl .hljs-sub, -pre .javascript .hljs-title, -code .javascript .hljs-title, -pre .coffeescript .hljs-title, -code .coffeescript .hljs-title { - color: #4271ae; -} -pre .hljs-keyword, -code .hljs-keyword, -pre .javascript .hljs-function, -code .javascript .hljs-function { - color: #8959a8; -} -pre .hljs, -code .hljs { - display: block; - background: white; - color: #4d4d4c; - padding: 0.5em; -} -pre .coffeescript .javascript, -code .coffeescript .javascript, -pre .javascript .xml, -code .javascript .xml, -pre .tex .hljs-formula, -code .tex .hljs-formula, -pre .xml .javascript, -code .xml .javascript, -pre .xml .vbscript, -code .xml .vbscript, -pre .xml .css, -code .xml .css, -pre .xml .hljs-cdata, -code .xml .hljs-cdata { - opacity: 0.5; -} diff --git a/docs/_book/gitbook/gitbook-plugin-lunr/lunr.min 2.js b/docs/_book/gitbook/gitbook-plugin-lunr/lunr.min 2.js deleted file mode 100644 index 6aa6bc7d6..000000000 --- a/docs/_book/gitbook/gitbook-plugin-lunr/lunr.min 2.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 0.5.12 - * Copyright (C) 2015 Oliver Nightingale - * MIT Licensed - * @license - */ -!function(){var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.5.12",t.utils={},t.utils.warn=function(t){return function(e){t.console&&console.warn&&console.warn(e)}}(this),t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var t=Array.prototype.slice.call(arguments),e=t.pop(),n=t;if("function"!=typeof e)throw new TypeError("last argument must be a function");n.forEach(function(t){this.hasHandler(t)||(this.events[t]=[]),this.events[t].push(e)},this)},t.EventEmitter.prototype.removeListener=function(t,e){if(this.hasHandler(t)){var n=this.events[t].indexOf(e);this.events[t].splice(n,1),this.events[t].length||delete this.events[t]}},t.EventEmitter.prototype.emit=function(t){if(this.hasHandler(t)){var e=Array.prototype.slice.call(arguments,1);this.events[t].forEach(function(t){t.apply(void 0,e)})}},t.EventEmitter.prototype.hasHandler=function(t){return t in this.events},t.tokenizer=function(t){return arguments.length&&null!=t&&void 0!=t?Array.isArray(t)?t.map(function(t){return t.toLowerCase()}):t.toString().trim().toLowerCase().split(/[\s\-]+/):[]},t.Pipeline=function(){this._stack=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in this.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[e.label]=e},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.registeredFunctions[e];if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._stack.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._stack.indexOf(e);if(-1==i)throw new Error("Cannot find existingFn");i+=1,this._stack.splice(i,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._stack.indexOf(e);if(-1==i)throw new Error("Cannot find existingFn");this._stack.splice(i,0,n)},t.Pipeline.prototype.remove=function(t){var e=this._stack.indexOf(t);-1!=e&&this._stack.splice(e,1)},t.Pipeline.prototype.run=function(t){for(var e=[],n=t.length,i=this._stack.length,o=0;n>o;o++){for(var r=t[o],s=0;i>s&&(r=this._stack[s](r,o,t),void 0!==r);s++);void 0!==r&&e.push(r)}return e},t.Pipeline.prototype.reset=function(){this._stack=[]},t.Pipeline.prototype.toJSON=function(){return this._stack.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Vector=function(){this._magnitude=null,this.list=void 0,this.length=0},t.Vector.Node=function(t,e,n){this.idx=t,this.val=e,this.next=n},t.Vector.prototype.insert=function(e,n){this._magnitude=void 0;var i=this.list;if(!i)return this.list=new t.Vector.Node(e,n,i),this.length++;if(en.idx?n=n.next:(i+=e.val*n.val,e=e.next,n=n.next);return i},t.Vector.prototype.similarity=function(t){return this.dot(t)/(this.magnitude()*t.magnitude())},t.SortedSet=function(){this.length=0,this.elements=[]},t.SortedSet.load=function(t){var e=new this;return e.elements=t,e.length=t.length,e},t.SortedSet.prototype.add=function(){var t,e;for(t=0;t1;){if(r===t)return o;t>r&&(e=o),r>t&&(n=o),i=n-e,o=e+Math.floor(i/2),r=this.elements[o]}return r===t?o:-1},t.SortedSet.prototype.locationFor=function(t){for(var e=0,n=this.elements.length,i=n-e,o=e+Math.floor(i/2),r=this.elements[o];i>1;)t>r&&(e=o),r>t&&(n=o),i=n-e,o=e+Math.floor(i/2),r=this.elements[o];return r>t?o:t>r?o+1:void 0},t.SortedSet.prototype.intersect=function(e){for(var n=new t.SortedSet,i=0,o=0,r=this.length,s=e.length,a=this.elements,h=e.elements;;){if(i>r-1||o>s-1)break;a[i]!==h[o]?a[i]h[o]&&o++:(n.add(a[i]),i++,o++)}return n},t.SortedSet.prototype.clone=function(){var e=new t.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},t.SortedSet.prototype.union=function(t){var e,n,i;return this.length>=t.length?(e=this,n=t):(e=t,n=this),i=e.clone(),i.add.apply(i,n.toArray()),i},t.SortedSet.prototype.toJSON=function(){return this.toArray()},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.Store,this.tokenStore=new t.TokenStore,this.corpusTokens=new t.SortedSet,this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var t=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,t)},t.Index.prototype.off=function(t,e){return this.eventEmitter.removeListener(t,e)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;return n._fields=e.fields,n._ref=e.ref,n.documentStore=t.Store.load(e.documentStore),n.tokenStore=t.TokenStore.load(e.tokenStore),n.corpusTokens=t.SortedSet.load(e.corpusTokens),n.pipeline=t.Pipeline.load(e.pipeline),n},t.Index.prototype.field=function(t,e){var e=e||{},n={name:t,boost:e.boost||1};return this._fields.push(n),this},t.Index.prototype.ref=function(t){return this._ref=t,this},t.Index.prototype.add=function(e,n){var i={},o=new t.SortedSet,r=e[this._ref],n=void 0===n?!0:n;this._fields.forEach(function(n){var r=this.pipeline.run(t.tokenizer(e[n.name]));i[n.name]=r,t.SortedSet.prototype.add.apply(o,r)},this),this.documentStore.set(r,o),t.SortedSet.prototype.add.apply(this.corpusTokens,o.toArray());for(var s=0;s0&&(i=1+Math.log(this.documentStore.length/n)),this._idfCache[e]=i},t.Index.prototype.search=function(e){var n=this.pipeline.run(t.tokenizer(e)),i=new t.Vector,o=[],r=this._fields.reduce(function(t,e){return t+e.boost},0),s=n.some(function(t){return this.tokenStore.has(t)},this);if(!s)return[];n.forEach(function(e,n,s){var a=1/s.length*this._fields.length*r,h=this,l=this.tokenStore.expand(e).reduce(function(n,o){var r=h.corpusTokens.indexOf(o),s=h.idf(o),l=1,u=new t.SortedSet;if(o!==e){var c=Math.max(3,o.length-e.length);l=1/Math.log(c)}return r>-1&&i.insert(r,a*s*l),Object.keys(h.tokenStore.get(o)).forEach(function(t){u.add(t)}),n.union(u)},new t.SortedSet);o.push(l)},this);var a=o.reduce(function(t,e){return t.intersect(e)});return a.map(function(t){return{ref:t,score:i.similarity(this.documentVector(t))}},this).sort(function(t,e){return e.score-t.score})},t.Index.prototype.documentVector=function(e){for(var n=this.documentStore.get(e),i=n.length,o=new t.Vector,r=0;i>r;r++){var s=n.elements[r],a=this.tokenStore.get(s)[e].tf,h=this.idf(s);o.insert(this.corpusTokens.indexOf(s),a*h)}return o},t.Index.prototype.toJSON=function(){return{version:t.version,fields:this._fields,ref:this._ref,documentStore:this.documentStore.toJSON(),tokenStore:this.tokenStore.toJSON(),corpusTokens:this.corpusTokens.toJSON(),pipeline:this.pipeline.toJSON()}},t.Index.prototype.use=function(t){var e=Array.prototype.slice.call(arguments,1);e.unshift(this),t.apply(this,e)},t.Store=function(){this.store={},this.length=0},t.Store.load=function(e){var n=new this;return n.length=e.length,n.store=Object.keys(e.store).reduce(function(n,i){return n[i]=t.SortedSet.load(e.store[i]),n},{}),n},t.Store.prototype.set=function(t,e){this.has(t)||this.length++,this.store[t]=e},t.Store.prototype.get=function(t){return this.store[t]},t.Store.prototype.has=function(t){return t in this.store},t.Store.prototype.remove=function(t){this.has(t)&&(delete this.store[t],this.length--)},t.Store.prototype.toJSON=function(){return{store:this.store,length:this.length}},t.stemmer=function(){var t={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},e={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",o=n+"[^aeiouy]*",r=i+"[aeiou]*",s="^("+o+")?"+r+o,a="^("+o+")?"+r+o+"("+r+")?$",h="^("+o+")?"+r+o+r+o,l="^("+o+")?"+i,u=new RegExp(s),c=new RegExp(h),f=new RegExp(a),d=new RegExp(l),p=/^(.+?)(ss|i)es$/,m=/^(.+?)([^s])s$/,v=/^(.+?)eed$/,y=/^(.+?)(ed|ing)$/,g=/.$/,S=/(at|bl|iz)$/,w=new RegExp("([^aeiouylsz])\\1$"),x=new RegExp("^"+o+i+"[^aeiouwxy]$"),k=/^(.+?[^aeiou])y$/,b=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,E=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,_=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,F=/^(.+?)(s|t)(ion)$/,O=/^(.+?)e$/,P=/ll$/,N=new RegExp("^"+o+i+"[^aeiouwxy]$"),T=function(n){var i,o,r,s,a,h,l;if(n.length<3)return n;if(r=n.substr(0,1),"y"==r&&(n=r.toUpperCase()+n.substr(1)),s=p,a=m,s.test(n)?n=n.replace(s,"$1$2"):a.test(n)&&(n=n.replace(a,"$1$2")),s=v,a=y,s.test(n)){var T=s.exec(n);s=u,s.test(T[1])&&(s=g,n=n.replace(s,""))}else if(a.test(n)){var T=a.exec(n);i=T[1],a=d,a.test(i)&&(n=i,a=S,h=w,l=x,a.test(n)?n+="e":h.test(n)?(s=g,n=n.replace(s,"")):l.test(n)&&(n+="e"))}if(s=k,s.test(n)){var T=s.exec(n);i=T[1],n=i+"i"}if(s=b,s.test(n)){var T=s.exec(n);i=T[1],o=T[2],s=u,s.test(i)&&(n=i+t[o])}if(s=E,s.test(n)){var T=s.exec(n);i=T[1],o=T[2],s=u,s.test(i)&&(n=i+e[o])}if(s=_,a=F,s.test(n)){var T=s.exec(n);i=T[1],s=c,s.test(i)&&(n=i)}else if(a.test(n)){var T=a.exec(n);i=T[1]+T[2],a=c,a.test(i)&&(n=i)}if(s=O,s.test(n)){var T=s.exec(n);i=T[1],s=c,a=f,h=N,(s.test(i)||a.test(i)&&!h.test(i))&&(n=i)}return s=P,a=c,s.test(n)&&a.test(n)&&(s=g,n=n.replace(s,"")),"y"==r&&(n=r.toLowerCase()+n.substr(1)),n};return T}(),t.Pipeline.registerFunction(t.stemmer,"stemmer"),t.stopWordFilter=function(e){return e&&t.stopWordFilter.stopWords[e]!==e?e:void 0},t.stopWordFilter.stopWords={a:"a",able:"able",about:"about",across:"across",after:"after",all:"all",almost:"almost",also:"also",am:"am",among:"among",an:"an",and:"and",any:"any",are:"are",as:"as",at:"at",be:"be",because:"because",been:"been",but:"but",by:"by",can:"can",cannot:"cannot",could:"could",dear:"dear",did:"did","do":"do",does:"does",either:"either","else":"else",ever:"ever",every:"every","for":"for",from:"from",get:"get",got:"got",had:"had",has:"has",have:"have",he:"he",her:"her",hers:"hers",him:"him",his:"his",how:"how",however:"however",i:"i","if":"if","in":"in",into:"into",is:"is",it:"it",its:"its",just:"just",least:"least",let:"let",like:"like",likely:"likely",may:"may",me:"me",might:"might",most:"most",must:"must",my:"my",neither:"neither",no:"no",nor:"nor",not:"not",of:"of",off:"off",often:"often",on:"on",only:"only",or:"or",other:"other",our:"our",own:"own",rather:"rather",said:"said",say:"say",says:"says",she:"she",should:"should",since:"since",so:"so",some:"some",than:"than",that:"that",the:"the",their:"their",them:"them",then:"then",there:"there",these:"these",they:"they","this":"this",tis:"tis",to:"to",too:"too",twas:"twas",us:"us",wants:"wants",was:"was",we:"we",were:"were",what:"what",when:"when",where:"where",which:"which","while":"while",who:"who",whom:"whom",why:"why",will:"will","with":"with",would:"would",yet:"yet",you:"you",your:"your"},t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter"),t.trimmer=function(t){var e=t.replace(/^\W+/,"").replace(/\W+$/,"");return""===e?void 0:e},t.Pipeline.registerFunction(t.trimmer,"trimmer"),t.TokenStore=function(){this.root={docs:{}},this.length=0},t.TokenStore.load=function(t){var e=new this;return e.root=t.root,e.length=t.length,e},t.TokenStore.prototype.add=function(t,e,n){var n=n||this.root,i=t[0],o=t.slice(1);return i in n||(n[i]={docs:{}}),0===o.length?(n[i].docs[e.ref]=e,void(this.length+=1)):this.add(o,e,n[i])},t.TokenStore.prototype.has=function(t){if(!t)return!1;for(var e=this.root,n=0;no;o++){for(var r=t[o],s=0;i>s&&(r=this._stack[s](r,o,t),void 0!==r);s++);void 0!==r&&e.push(r)}return e},t.Pipeline.prototype.reset=function(){this._stack=[]},t.Pipeline.prototype.toJSON=function(){return this._stack.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Vector=function(){this._magnitude=null,this.list=void 0,this.length=0},t.Vector.Node=function(t,e,n){this.idx=t,this.val=e,this.next=n},t.Vector.prototype.insert=function(e,n){this._magnitude=void 0;var i=this.list;if(!i)return this.list=new t.Vector.Node(e,n,i),this.length++;if(en.idx?n=n.next:(i+=e.val*n.val,e=e.next,n=n.next);return i},t.Vector.prototype.similarity=function(t){return this.dot(t)/(this.magnitude()*t.magnitude())},t.SortedSet=function(){this.length=0,this.elements=[]},t.SortedSet.load=function(t){var e=new this;return e.elements=t,e.length=t.length,e},t.SortedSet.prototype.add=function(){var t,e;for(t=0;t1;){if(r===t)return o;t>r&&(e=o),r>t&&(n=o),i=n-e,o=e+Math.floor(i/2),r=this.elements[o]}return r===t?o:-1},t.SortedSet.prototype.locationFor=function(t){for(var e=0,n=this.elements.length,i=n-e,o=e+Math.floor(i/2),r=this.elements[o];i>1;)t>r&&(e=o),r>t&&(n=o),i=n-e,o=e+Math.floor(i/2),r=this.elements[o];return r>t?o:t>r?o+1:void 0},t.SortedSet.prototype.intersect=function(e){for(var n=new t.SortedSet,i=0,o=0,r=this.length,s=e.length,a=this.elements,h=e.elements;;){if(i>r-1||o>s-1)break;a[i]!==h[o]?a[i]h[o]&&o++:(n.add(a[i]),i++,o++)}return n},t.SortedSet.prototype.clone=function(){var e=new t.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},t.SortedSet.prototype.union=function(t){var e,n,i;return this.length>=t.length?(e=this,n=t):(e=t,n=this),i=e.clone(),i.add.apply(i,n.toArray()),i},t.SortedSet.prototype.toJSON=function(){return this.toArray()},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.Store,this.tokenStore=new t.TokenStore,this.corpusTokens=new t.SortedSet,this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var t=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,t)},t.Index.prototype.off=function(t,e){return this.eventEmitter.removeListener(t,e)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;return n._fields=e.fields,n._ref=e.ref,n.documentStore=t.Store.load(e.documentStore),n.tokenStore=t.TokenStore.load(e.tokenStore),n.corpusTokens=t.SortedSet.load(e.corpusTokens),n.pipeline=t.Pipeline.load(e.pipeline),n},t.Index.prototype.field=function(t,e){var e=e||{},n={name:t,boost:e.boost||1};return this._fields.push(n),this},t.Index.prototype.ref=function(t){return this._ref=t,this},t.Index.prototype.add=function(e,n){var i={},o=new t.SortedSet,r=e[this._ref],n=void 0===n?!0:n;this._fields.forEach(function(n){var r=this.pipeline.run(t.tokenizer(e[n.name]));i[n.name]=r,t.SortedSet.prototype.add.apply(o,r)},this),this.documentStore.set(r,o),t.SortedSet.prototype.add.apply(this.corpusTokens,o.toArray());for(var s=0;s0&&(i=1+Math.log(this.documentStore.length/n)),this._idfCache[e]=i},t.Index.prototype.search=function(e){var n=this.pipeline.run(t.tokenizer(e)),i=new t.Vector,o=[],r=this._fields.reduce(function(t,e){return t+e.boost},0),s=n.some(function(t){return this.tokenStore.has(t)},this);if(!s)return[];n.forEach(function(e,n,s){var a=1/s.length*this._fields.length*r,h=this,l=this.tokenStore.expand(e).reduce(function(n,o){var r=h.corpusTokens.indexOf(o),s=h.idf(o),l=1,u=new t.SortedSet;if(o!==e){var c=Math.max(3,o.length-e.length);l=1/Math.log(c)}return r>-1&&i.insert(r,a*s*l),Object.keys(h.tokenStore.get(o)).forEach(function(t){u.add(t)}),n.union(u)},new t.SortedSet);o.push(l)},this);var a=o.reduce(function(t,e){return t.intersect(e)});return a.map(function(t){return{ref:t,score:i.similarity(this.documentVector(t))}},this).sort(function(t,e){return e.score-t.score})},t.Index.prototype.documentVector=function(e){for(var n=this.documentStore.get(e),i=n.length,o=new t.Vector,r=0;i>r;r++){var s=n.elements[r],a=this.tokenStore.get(s)[e].tf,h=this.idf(s);o.insert(this.corpusTokens.indexOf(s),a*h)}return o},t.Index.prototype.toJSON=function(){return{version:t.version,fields:this._fields,ref:this._ref,documentStore:this.documentStore.toJSON(),tokenStore:this.tokenStore.toJSON(),corpusTokens:this.corpusTokens.toJSON(),pipeline:this.pipeline.toJSON()}},t.Index.prototype.use=function(t){var e=Array.prototype.slice.call(arguments,1);e.unshift(this),t.apply(this,e)},t.Store=function(){this.store={},this.length=0},t.Store.load=function(e){var n=new this;return n.length=e.length,n.store=Object.keys(e.store).reduce(function(n,i){return n[i]=t.SortedSet.load(e.store[i]),n},{}),n},t.Store.prototype.set=function(t,e){this.has(t)||this.length++,this.store[t]=e},t.Store.prototype.get=function(t){return this.store[t]},t.Store.prototype.has=function(t){return t in this.store},t.Store.prototype.remove=function(t){this.has(t)&&(delete this.store[t],this.length--)},t.Store.prototype.toJSON=function(){return{store:this.store,length:this.length}},t.stemmer=function(){var t={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},e={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",o=n+"[^aeiouy]*",r=i+"[aeiou]*",s="^("+o+")?"+r+o,a="^("+o+")?"+r+o+"("+r+")?$",h="^("+o+")?"+r+o+r+o,l="^("+o+")?"+i,u=new RegExp(s),c=new RegExp(h),f=new RegExp(a),d=new RegExp(l),p=/^(.+?)(ss|i)es$/,m=/^(.+?)([^s])s$/,v=/^(.+?)eed$/,y=/^(.+?)(ed|ing)$/,g=/.$/,S=/(at|bl|iz)$/,w=new RegExp("([^aeiouylsz])\\1$"),x=new RegExp("^"+o+i+"[^aeiouwxy]$"),k=/^(.+?[^aeiou])y$/,b=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,E=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,_=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,F=/^(.+?)(s|t)(ion)$/,O=/^(.+?)e$/,P=/ll$/,N=new RegExp("^"+o+i+"[^aeiouwxy]$"),T=function(n){var i,o,r,s,a,h,l;if(n.length<3)return n;if(r=n.substr(0,1),"y"==r&&(n=r.toUpperCase()+n.substr(1)),s=p,a=m,s.test(n)?n=n.replace(s,"$1$2"):a.test(n)&&(n=n.replace(a,"$1$2")),s=v,a=y,s.test(n)){var T=s.exec(n);s=u,s.test(T[1])&&(s=g,n=n.replace(s,""))}else if(a.test(n)){var T=a.exec(n);i=T[1],a=d,a.test(i)&&(n=i,a=S,h=w,l=x,a.test(n)?n+="e":h.test(n)?(s=g,n=n.replace(s,"")):l.test(n)&&(n+="e"))}if(s=k,s.test(n)){var T=s.exec(n);i=T[1],n=i+"i"}if(s=b,s.test(n)){var T=s.exec(n);i=T[1],o=T[2],s=u,s.test(i)&&(n=i+t[o])}if(s=E,s.test(n)){var T=s.exec(n);i=T[1],o=T[2],s=u,s.test(i)&&(n=i+e[o])}if(s=_,a=F,s.test(n)){var T=s.exec(n);i=T[1],s=c,s.test(i)&&(n=i)}else if(a.test(n)){var T=a.exec(n);i=T[1]+T[2],a=c,a.test(i)&&(n=i)}if(s=O,s.test(n)){var T=s.exec(n);i=T[1],s=c,a=f,h=N,(s.test(i)||a.test(i)&&!h.test(i))&&(n=i)}return s=P,a=c,s.test(n)&&a.test(n)&&(s=g,n=n.replace(s,"")),"y"==r&&(n=r.toLowerCase()+n.substr(1)),n};return T}(),t.Pipeline.registerFunction(t.stemmer,"stemmer"),t.stopWordFilter=function(e){return e&&t.stopWordFilter.stopWords[e]!==e?e:void 0},t.stopWordFilter.stopWords={a:"a",able:"able",about:"about",across:"across",after:"after",all:"all",almost:"almost",also:"also",am:"am",among:"among",an:"an",and:"and",any:"any",are:"are",as:"as",at:"at",be:"be",because:"because",been:"been",but:"but",by:"by",can:"can",cannot:"cannot",could:"could",dear:"dear",did:"did","do":"do",does:"does",either:"either","else":"else",ever:"ever",every:"every","for":"for",from:"from",get:"get",got:"got",had:"had",has:"has",have:"have",he:"he",her:"her",hers:"hers",him:"him",his:"his",how:"how",however:"however",i:"i","if":"if","in":"in",into:"into",is:"is",it:"it",its:"its",just:"just",least:"least",let:"let",like:"like",likely:"likely",may:"may",me:"me",might:"might",most:"most",must:"must",my:"my",neither:"neither",no:"no",nor:"nor",not:"not",of:"of",off:"off",often:"often",on:"on",only:"only",or:"or",other:"other",our:"our",own:"own",rather:"rather",said:"said",say:"say",says:"says",she:"she",should:"should",since:"since",so:"so",some:"some",than:"than",that:"that",the:"the",their:"their",them:"them",then:"then",there:"there",these:"these",they:"they","this":"this",tis:"tis",to:"to",too:"too",twas:"twas",us:"us",wants:"wants",was:"was",we:"we",were:"were",what:"what",when:"when",where:"where",which:"which","while":"while",who:"who",whom:"whom",why:"why",will:"will","with":"with",would:"would",yet:"yet",you:"you",your:"your"},t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter"),t.trimmer=function(t){var e=t.replace(/^\W+/,"").replace(/\W+$/,"");return""===e?void 0:e},t.Pipeline.registerFunction(t.trimmer,"trimmer"),t.TokenStore=function(){this.root={docs:{}},this.length=0},t.TokenStore.load=function(t){var e=new this;return e.root=t.root,e.length=t.length,e},t.TokenStore.prototype.add=function(t,e,n){var n=n||this.root,i=t[0],o=t.slice(1);return i in n||(n[i]={docs:{}}),0===o.length?(n[i].docs[e.ref]=e,void(this.length+=1)):this.add(o,e,n[i])},t.TokenStore.prototype.has=function(t){if(!t)return!1;for(var e=this.root,n=0;n 0) { - gitbook.toolbar.createButton({ - icon: 'fa fa-share-alt', - label: 'Share', - position: 'right', - dropdown: [menu] - }); - } - - // Direct actions to share - $.each(SITES, function(sideId, site) { - if (!opts[sideId]) return; - - gitbook.toolbar.createButton({ - icon: site.icon, - label: site.text, - position: 'right', - onClick: site.onClick - }); - }); - }); -}); diff --git a/docs/_book/gitbook/images/apple-touch-icon-precomposed-152 2.png b/docs/_book/gitbook/images/apple-touch-icon-precomposed-152 2.png deleted file mode 100644 index f0adcfd0b..000000000 Binary files a/docs/_book/gitbook/images/apple-touch-icon-precomposed-152 2.png and /dev/null differ diff --git a/docs/_book/index.html b/docs/_book/index.html index f38301841..02bbe10d3 100644 --- a/docs/_book/index.html +++ b/docs/_book/index.html @@ -548,7 +548,7 @@

          No results matching " var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":".","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":".","book":{"language":""}}); }); @@ -558,6 +558,10 @@

          No results matching " + + + + diff --git a/frontend/docs/file-manager-prototype.png b/frontend/docs/file-manager-prototype.png deleted file mode 100644 index 270435b7e..000000000 Binary files a/frontend/docs/file-manager-prototype.png and /dev/null differ diff --git a/frontend/src/api/login.js b/frontend/src/api/login.js deleted file mode 100644 index 4699f07e9..000000000 --- a/frontend/src/api/login.js +++ /dev/null @@ -1,27 +0,0 @@ -import request from '@/utils/request' - -export function login (username, password) { - return request({ - url: '/user/login', - method: 'post', - data: { - username, - password - } - }) -} - -export function getInfo (token) { - return request({ - url: '/user/info', - method: 'get', - params: { token } - }) -} - -export function logout () { - return request({ - url: '/user/logout', - method: 'post' - }) -} diff --git a/frontend/src/api/table.js b/frontend/src/api/table.js deleted file mode 100644 index 95d1df604..000000000 --- a/frontend/src/api/table.js +++ /dev/null @@ -1,9 +0,0 @@ -import request from '@/utils/request' - -export function getList (params) { - return request({ - url: '/table/list', - method: 'get', - params - }) -} diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue new file mode 100644 index 000000000..6c47570a2 --- /dev/null +++ b/frontend/src/components/Config/ConfigList.vue @@ -0,0 +1,275 @@ + + + + + diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index fe384f149..e896bbc0a 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -12,10 +12,10 @@ - + - + @@ -32,13 +32,12 @@ - - - - + + + - + @@ -49,8 +48,8 @@ - {{$t('Run')}} - {{$t('Deploy')}} + {{$t('Run')}} + {{$t('Deploy')}} {{$t('Save')}} @@ -99,17 +98,23 @@ export default { 'spiderForm' ]), isShowRun () { - if (!this.spiderForm.deploy_ts) { - return false - } - if (!this.spiderForm.cmd) { - return false + if (this.isCustomized) { + // customized spider + if (!this.spiderForm.deploy_ts) { + return false + } + return !!this.spiderForm.cmd + } else { + // configurable spider + return !!this.spiderForm.fields } - return true + }, + isCustomized () { + return this.spiderForm.type === 'customized' } }, methods: { - onRun () { + onCrawl () { const row = this.spiderForm this.$refs['spiderForm'].validate(res => { if (res) { diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue new file mode 100644 index 000000000..ab2376f7f --- /dev/null +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -0,0 +1,179 @@ + + + + + diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index dd1d529eb..e6db4795c 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -13,16 +13,18 @@ export default { 'Sites': '网站', // 标签 - Overview: '概览', - Files: '文件', + 'Overview': '概览', + 'Files': '文件', 'Deployed Spiders': '已部署爬虫', 'Log': '日志', 'Results': '结果', 'Environment': '环境', 'Analytics': '分析', + 'Rules': '规则', + 'Config': '配置', // 选择 - Spider: '爬虫', + 'Spider': '爬虫', // 块标题 'Latest Tasks': '最近任务', @@ -37,6 +39,7 @@ export default { REVOKED: '已取消', // 操作 + Add: '添加', Run: '运行', Deploy: '部署', Save: '保存', @@ -51,6 +54,7 @@ export default { Remove: '删除', Confirm: '确认', Stop: '停止', + Preview: '预览', // 主页 'Total Tasks': '总任务数', @@ -88,6 +92,10 @@ export default { 'Variable': '变量', 'Value': '值', 'Add Environment Variables': '添加环境变量', + 'Add Spider': '添加爬虫', + 'Add Configurable Spider': '添加可配置爬虫', + 'Add Customized Spider': '添加自定义爬虫', + 'Add Field': '添加字段', 'Last 7-Day Tasks': '最近7天任务数', 'Last 5-Run Errors': '最近5次运行错误数', '30-Day Tasks': '最近30天任务数', @@ -98,6 +106,31 @@ export default { 'Tasks by Node': '分节点任务数', 'Daily Tasks': '每日任务数', 'Daily Avg Duration (sec)': '每日平均运行时长(秒)', + 'Configurable Spider': '可配置爬虫', + 'Customized Spider': '自定义爬虫', + 'Configurable': '可配置', + 'Customized': '自定义', + 'Text': '文本', + 'Attribute': '属性', + 'Field Name': '字段名称', + 'Query Type': '查询类别', + 'Query': '查询', + 'Extract Type': '提取类别', + 'CSS Selector': 'CSS选择器', + 'Crawl Type': '抓取类别', + 'List Only': '仅列表', + 'Detail Only': '仅详情页', + 'List + Detail': '列表＋详情页', + 'Start URL': '开始URL', + 'Item Selector': '列表项选择器', + 'Item Selector Type': '列表项选择器类别', + 'Pagination Selector': '分页选择器', + 'Pagination Selector Type': '分页项选择器类别', + 'Preview Results': '预览结果', + 'Obey robots.txt': '遵守Robots协议', + 'List Page Fields': '列表页字段', + 'Detail Page Fields': '详情页字段', + 'Detail Page URL': '详情页URL', // 爬虫列表 'Name': '名称', @@ -135,10 +168,15 @@ export default { 'Site': '网站', 'Rank': '排名', 'Domain': '域名', + 'Main Category': '主类别', 'Category': '类别', 'Select': '请选择', + 'Select Main Category': '请选择主类别', 'Select Category': '请选择类别', 'Spider Count': '爬虫数', + 'Robots Protocol': 'Robots 协议', + 'Home Page Response Time (sec)': '首页响应时间(秒)', + 'Home Page Response Status Code': '首页响应状态码', // 文件 'Choose Folder': '选择文件', diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 33ae9f404..57fb5065c 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -1,7 +1,7 @@ import Vue from 'vue' import Vuex from 'vuex' import app from './modules/app' -import user from './modules/user' +// import user from './modules/user' import tagsView from './modules/tagsView' import dialogView from './modules/dialogView' import node from './modules/node' @@ -19,7 +19,7 @@ Vue.use(Vuex) const store = new Vuex.Store({ modules: { app, - user, + // user, tagsView, dialogView, node, diff --git a/frontend/src/store/modules/site.js b/frontend/src/store/modules/site.js index d05b8de4d..ec1de9f33 100644 --- a/frontend/src/store/modules/site.js +++ b/frontend/src/store/modules/site.js @@ -1,10 +1,18 @@ import request from '../../api/request' const state = { + // site list siteList: [], + // main category list + mainCategoryList: [], + + // (sub) category list + categoryList: [], + // filter filter: { + mainCategory: undefined, category: undefined }, keyword: '', @@ -32,6 +40,12 @@ const mutations = { }, SET_TOTAL_COUNT (state, value) { state.totalCount = value + }, + SET_MAIN_CATEGORY_LIST (state, value) { + state.mainCategoryList = value + }, + SET_CATEGORY_LIST (state, value) { + state.categoryList = value } } @@ -48,6 +62,7 @@ const actions = { page_size: state.pageSize, keyword: state.keyword || undefined, filter: { + main_category: state.filter.mainCategory || undefined, category: state.filter.category || undefined } }) @@ -55,6 +70,20 @@ const actions = { commit('SET_SITE_LIST', response.data.items) commit('SET_TOTAL_COUNT', response.data.total_count) }) + }, + getMainCategoryList ({ state, commit }) { + return request.get('/sites/get/get_main_category_list') + .then(response => { + commit('SET_MAIN_CATEGORY_LIST', response.data.items) + }) + }, + getCategoryList ({ state, commit }) { + return request.get('/sites/get/get_category_list', { + 'main_category': state.filter.mainCategory || undefined + }) + .then(response => { + commit('SET_CATEGORY_LIST', response.data.items) + }) } } diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index b7e702fd5..673ce33fb 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -26,7 +26,13 @@ const state = { dailyStats: [], // spider node stats - nodeStats: [] + nodeStats: [], + + // filters + filterSite: '', + + // preview crawl data + previewCrawlData: [] } const getters = {} @@ -55,12 +61,23 @@ const mutations = { }, SET_NODE_STATS (state, value) { state.nodeStats = value + }, + SET_FILTER_SITE (state, value) { + state.filterSite = value + }, + SET_PREVIEW_CRAWL_DATA (state, value) { + state.previewCrawlData = value } } const actions = { getSpiderList ({ state, commit }) { - return request.get('/spiders', {}) + let params = {} + if (state.filterSite) { + params.site = state.filterSite + } + console.log(params) + return request.get('/spiders', params) .then(response => { commit('SET_SPIDER_LIST', response.data.items) }) @@ -68,13 +85,8 @@ const actions = { addSpider ({ state, dispatch }) { return request.put('/spiders', { name: state.spiderForm.name, - src: state.spiderForm.src, - cmd: state.spiderForm.cmd, - type: state.spiderForm.type, - lang: state.spiderForm.lang, col: state.spiderForm.col, - cron: state.spiderForm.cron, - cron_enabled: state.spiderForm.cron_enabled ? 1 : 0, + type: 'configurable', site: state.spiderForm.site }) .then(() => { @@ -89,9 +101,15 @@ const actions = { type: state.spiderForm.type, lang: state.spiderForm.lang, col: state.spiderForm.col, - cron: state.spiderForm.cron, - cron_enabled: state.spiderForm.cron_enabled ? 1 : 0, - site: state.spiderForm.site + site: state.spiderForm.site, + // configurable spider + crawl_type: state.spiderForm.crawl_type, + start_url: state.spiderForm.start_url, + item_selector: state.spiderForm.item_selector, + item_selector_type: state.spiderForm.item_selector_type, + pagination_selector: state.spiderForm.pagination_selector, + pagination_selector_type: state.spiderForm.pagination_selector_type, + obey_robots_txt: state.spiderForm.obey_robots_txt }) .then(() => { dispatch('getSpiderList') @@ -108,6 +126,16 @@ const actions = { envs: JSON.stringify(state.spiderForm.envs) }) }, + updateSpiderFields ({ state }) { + return request.post(`/spiders/${state.spiderForm._id}/update_fields`, { + fields: JSON.stringify(state.spiderForm.fields) + }) + }, + updateSpiderDetailFields ({ state }) { + return request.post(`/spiders/${state.spiderForm._id}/update_detail_fields`, { + detail_fields: JSON.stringify(state.spiderForm.detail_fields) + }) + }, getSpiderData ({ state, commit }, id) { return request.get(`/spiders/${id}`) .then(response => { @@ -173,6 +201,12 @@ const actions = { commit('SET_DAILY_STATS', response.data.daily_stats) commit('SET_NODE_STATS', response.data.task_count_by_node) }) + }, + getPreviewCrawlData ({ state, commit }) { + return request.post(`/spiders/${state.spiderForm._id}/preview_crawl`) + .then(response => { + commit('SET_PREVIEW_CRAWL_DATA', response.data.items) + }) } } diff --git a/frontend/src/views/layout/Layout.vue b/frontend/src/views/layout/Layout.vue index bf284ee6c..2b182d31c 100644 --- a/frontend/src/views/layout/Layout.vue +++ b/frontend/src/views/layout/Layout.vue @@ -4,7 +4,7 @@
          - +
          @@ -14,8 +14,8 @@ import { Navbar, Sidebar, - AppMain - // TagsView + AppMain, + TagsView } from './components' import ResizeMixin from './mixin/ResizeHandler' @@ -24,7 +24,7 @@ export default { components: { Navbar, Sidebar, - // TagsView, + TagsView, AppMain }, mixins: [ResizeMixin], diff --git a/frontend/src/views/layout/components/TagsView.vue b/frontend/src/views/layout/components/TagsView.vue index 0f1c92aa4..406ed45c4 100644 --- a/frontend/src/views/layout/components/TagsView.vue +++ b/frontend/src/views/layout/components/TagsView.vue @@ -11,7 +11,7 @@ class="tags-view-item" @click.middle.native="closeSelectedTag(tag)" @contextmenu.prevent.native="openMenu(tag,$event)"> - {{ generateTitle(tag.title) }} + {{ $t(generateTitle(tag.title)) }} diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 7bcdcdf0b..e9fe0bce4 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -196,7 +196,7 @@ export default { }, 100) }) }, - onRun () { + onCrawl () { } }, created () { diff --git a/frontend/src/views/site/SiteList.vue b/frontend/src/views/site/SiteList.vue index 9c053dc5e..98fa13659 100644 --- a/frontend/src/views/site/SiteList.vue +++ b/frontend/src/views/site/SiteList.vue @@ -7,11 +7,16 @@ class="filter-search" v-model="keyword"> - + + + + {{$t('Search')}} @@ -21,35 +26,90 @@ - +