connect_mongdb_back

#!/usr/bin/python3
import re
import json
import pymongo
from itertools import groupby
from operator import itemgetter


def check_contain_chinese(check_str):
    for ch in check_str:
        if u'\u4e00' <= ch <= u'\u9fff':
            return True
    return False


def search_keywords(content):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["keyWordsContent"]
    res = []
    print(content)
    for i in mycol.find({"title": re.compile(content)}):
        tempMap= {'ID': i['ID'], 'title': i['title'], 'imgUrl':'', 'summarys': ''}
        tempMap['imgUrl']=i['imgUrl'][0:-5]+"140.jpg"
        for s in i['summary']:
            tempMap['summarys']+=s
        tempMap['summarys']=tempMap['summarys'][0:100]
        tempMap['summarys']+="..."
        res.append(tempMap)
        if len(res)>=20:
            break
    res = json.dumps(res, ensure_ascii=False)
    print(res)
    myclient.close()
    return res


def search_item(ID):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["keyWordsContent"]
    res=''
    for i in mycol.find({"ID": int(ID)}):
        temp=''
        del (i['_id'])
        for s in i['summary']:
            temp+=s
        i['summary']=temp
        print(i)
        res = i
    res = json.dumps(res, ensure_ascii=False)
    print(res)
    myclient.close()
    return res


def get_item():
    record_ID = get_current_ID()
    currentID = record_ID
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["keyWordsContent"]
    while not [x for x in mycol.find({"ID": currentID})]:
        currentID+=1
    res=''
    for i in mycol.find({"ID": currentID}):
        del (i['_id'])
        temp = ''
        for s in i['summary']:
            temp += s
        i['summary'] = temp
        res = i
    currentID += 1
    record_num(record_ID, currentID, res['title'])
    res = json.dumps(res, ensure_ascii=False)
    print(res)
    myclient.close()
    return res


def record_num(oldnum, num, title):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["recordNum"]
    #mycol.insert({"ID":1,"title":"深度学习","oldId":0})
    mycol.update({"ID": oldnum}, {'$set': {"ID": num, "title": title, "oldId": oldnum}})
    myclient.close()


def get_current_ID():
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["recordNum"]
    col = mycol.find()
    for C in col:
        ID = C['ID']
    myclient.close()
    return ID


def relation_label(keys):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["keyWordsContent"]
    sentence=[]
    count=0
    summary=[]
    for i in mycol.find():
        summary.extend(i['summary'])
    myclient.close()
    for S in summary:
        S.split('。')
        sentence.append(S)
    for S in sentence:
        entityList=[]
        for K in keys:
            if K in S:
                print(K)
                entityList.append(K)
        if entityList:
            print(entityList)
            recordEntity(S,entityList,count)
            print("__________处理个数_"+str(count)+"_____________")
            count += 1


def recordEntity(sentence,entity,count):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["sentenceEntity"]
    mycol.insert({"ID":count,"sentence":sentence,"entity":entity})
    myclient.close()


def get_keywords():
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["keyWordsContent"]
    keyWordsList = []
    for i in mycol.find():
        keys = i['title'].split("[")[0]
        if len(keys)>1:
            keyWordsList.append(keys)
    myclient.close()
    return keyWordsList


def recordLabel(entity,label,ID):
    entity = entity.split("[")[0]
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['BaiKe4']
    mycol = mydb["labeledEntity"]
    mycol.insert({"ID": ID, "entity": entity, "label": label})
    myclient.close()
    f = open('label', 'a+')
    f.write(entity+" "+label+"\n")
    f.close()
    res = {
        "state":"success"
    }
    res = json.dumps(res)
    return res


def getUpdateCount(flag):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb[flag]
    col = mycol.find()
    for C in col:
        count = C['count']
    mycol.update({"count": count}, {'$set': {"count": count+1}})
    myclient.close()
    return count


def getCount(flag):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb[flag]
    col = mycol.find()
    for C in col:
        count = C['count']
    myclient.close()
    return count


def recordLabelEntity(entity, entityType, sentence):
    ID = getUpdateCount("countEntity")
    ID += 1
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["labelEntity"]
    mycol.insert({"ID": ID, "entity":entity,"entityType":entityType, "sentence": sentence})
    myclient.close()
    return ID


def recordLabelRelation(entity1, entity2, relation, sentence):
    ID = getUpdateCount("countRelation")
    ID += 1
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["labelRelation"]
    mycol.insert({"ID": ID, "entity1": entity1, "relation": relation, "entity2": entity2, "sentence": sentence})
    myclient.close()
    return ID


def createSentence(dataSetName, dataSetColName, id):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    colName = "contents"+id
    mycol = mydb[colName]
    countColName = 'countContents' + id
    mycolCount = mydb[countColName]
    mycol.drop()
    mycolCount.drop()
    myclient.close()
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient[dataSetName]
    mycol = mydb[dataSetColName]
    col = mycol.find()
    sentenceList = list()
    for C in col:
        sentence = C['sentence']
        sentenceList.extend(sentence.split("。"))
    myclient.close()
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb[colName]
    count = 1
    for sentence in sentenceList:
        if sentence != '':
            mycol.insert({"ID": count, "sentence": sentence, "flag":0})
            count += 1
    myclient.close()
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    countColName='countContents'+id
    mycol = mydb[countColName]
    mycol.insert({"count": 0})
    myclient.close()
    print("重建数据集完成")
    return "success"


def loadSentence():
    ID = getUpdateCount("countContents")
    ID += 1
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["contents"]
    for i in mycol.find({"ID": ID}):
        sentence = i['sentence']
        ID = i['ID']
        mycol.update({"ID": ID}, {'$set': {"flag": 1}})
    myclient.close()
    print(sentence)
    res = {
        "sentence": sentence,
        "ID": ID
    }
    print(res)
    res = json.dumps(res, ensure_ascii=False)
    return res


def searchContext(content):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["contents"]
    count = 0
    for j in mycol.find({"sentence": re.compile(content)}):
        count += 1
    currentNum = 0
    sentence = ''
    ID = None
    for i in mycol.find({"sentence": re.compile(content)}):
        currentNum += 1
        flag = i['flag']
        if flag == 0:
            sentence = i['sentence']
            ID = i['ID']
            mycol.update({"ID": ID}, {'$set': {"flag": 1}})
            break
        print(sentence)
    resMap = {
        "sumCount": count,
        "currentNum": currentNum,
        "sentence": sentence,
        "ID": ID
    }
    res = json.dumps(resMap, ensure_ascii=False)
    myclient.close()
    return res


def loadLabeledDataSet(dataSetId):
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["labelEntity"]
    entityList = list()
    categoriesList = list()
    for i in mycol.find():
        tempMap = {
            "entity": i["entity"],
            "entity_type": i["entityType"],
            "sentence": i["sentence"]
        }
        entityList.append(tempMap)
        categoriesList.append(i["entityType"])
    myclient.close()
    myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    mydb = myclient['labelEntityRelation']
    mycol = mydb["labelRelation"]
    relationList = list()
    for i in mycol.find():
        tempMap = {
            "entity1": i["entity1"],
            "entity2": i["entity2"],
            "relation": i["relation"],
            "sentence": i["sentence"]
        }
        relationList.append(tempMap)
    myclient.close()
    entityList = distinct(entityList, "entity")
    categories = list(set(categoriesList))
    resultMap = {
        "entityList": entityList,
        "relationList": relationList,
        "categories": categories
    }
    print(resultMap)
    resultMap = json.dumps(resultMap, ensure_ascii=False)
    return resultMap


def distinct(items, key):
    key = itemgetter(key)
    items = sorted(items, key=key)
    return [next(v) for _, v in groupby(items, key=key)]


if __name__=="__main__":
    createSentence("BaiKe4", "sentenceEntity", "1")
    #createSentence()
    # myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    # mydb = myclient['BaiduXueshu2']
    # mycol = mydb["relatePaper"]
    # col = mycol.find()
    # sentenceList = list()
    # for C in col:
    #     sentence = C['abstract']
    #     if sentence:
    #         sentenceList.extend(sentence.split("。"))
    # myclient.close()
    # myclient = pymongo.MongoClient('mongodb://localhost:27017/')
    # mydb = myclient['BaiduXueshu']
    # mycol = mydb["abstract"]
    # count = 1
    # for sentence in sentenceList:
    #     if sentence != '':
    #         mycol.insert({"ID": count, "sentence": sentence})
    #         count += 1
    # myclient.close()