money-mining-python/analyse.py

# -*- coding:utf-8 -*-

import math
import jieba
import re
import os
import pickle

word_dict = {}

SPECIAL_SIMBOL = [".", "?", "^", "$", "*", "+", "\\", "[", "]", "|", "{", "}", "(", ")"]

def cut_word(word):
    """
    分词
    """
    word_root = jieba.cut_for_search(word)
    return list(word_root)

def merge_word_root(word_root_a, word_root_b):
    """
    合并词根
    """
    return list(set(word_root_a).union(set(word_root_b)))

def gen_word_vector(word_a, word_b, word_root_union):
    """
    生成词向量
    """
    a_word_vector, b_word_vector = [], []
    for word in word_root_union:
        if word in SPECIAL_SIMBOL :
            word = "\\" + word
        a_word_vector.append(len(re.findall(word, word_a)))
        b_word_vector.append(len(re.findall(word, word_b)))
    return a_word_vector, b_word_vector

def vector_multi(a_vector, b_vector):
    """
    向量相乘求和
    """
    return sum(map(lambda a_b: a_b[0]*a_b[1], zip(a_vector, b_vector)))

def vector_square_sum(word_vector):
    """
    向量平方求和
    """
    sum = 0
    for i in word_vector:
        sum = sum + i * i
    return sum

def vector_cos(v_multi, a_v_ss, b_v_ss):
    """
    计算余弦值
    """
    return v_multi / (math.sqrt(a_v_ss) * math.sqrt(b_v_ss))

def cal_cos(a_word, b_word, word_dict):
    """
    计算两个长尾关键词的余弦值
    """
    # a_word_root = cut_word(a_word)
    # b_word_root = cut_word(b_word)

    a_word_root = word_dict[a_word]
    b_word_root = word_dict[b_word]

    # 合并词根，用于生成词向量
    union_word_root = merge_word_root(a_word_root, b_word_root)

    # 生成词向量
    a_vector, b_vector = gen_word_vector(a_word, b_word, union_word_root)

    # 词向量相乘求和
    ab_vector_multi = vector_multi(a_vector, b_vector)

    # 向量平方求和
    a_vector_squar_sum = vector_square_sum(a_vector)
    b_vector_squar_sum = vector_square_sum(b_vector)

    cos_val = vector_cos(ab_vector_multi, a_vector_squar_sum, b_vector_squar_sum)

    return cos_val


def load_word_root_cache():
    word_root_cache = {}

    if os.path.exists("./data/pkl/word_root_cache.pkl"):
        print("存在缓存，开始加载")
        with open("./data/pkl/word_root_cache.pkl", "rb") as f:
            word_root_cache = pickle.load(f)
            return word_root_cache

    print('不存在缓存，开始构建分词字典')
    with open("./data/分词结果_bak.txt", "r", encoding="UTF-8") as f:
        lines = f.readlines()
        for line in lines:
            index = line.index(",")
            word_root_cache[line[:index]] = line[index+1:]
    
    print("构建完成，保存到本地")
    with open("./data/pkl/word_root_cache.pkl", "wb") as f:
        pickle.dump(word_root_cache, f)
    
    return word_root_cache

word_dict = load_word_root_cache()

key_list = list(word_dict.keys())
for i, a_key in enumerate(key_list[:-1]):
    with open("./data/category/%s.txt" % a_key, "w", encoding="UTF-8") as f:
        f.write(a_key)
        f.write("\n\n")

        del_container = []
        for j, b_key in  enumerate(key_list[i+1:]):
            if j % 100000 == 0 :
                print("正在处理：%d, %d" % (i, j))
            cos_val = cal_cos(a_key, b_key, word_dict)
            if cos_val > 0.8 :
                print("%s 与 %s 的余弦值：%f " % (a_key, b_key, cos_val))
                f.write(b_key)
                f.write("\n")
        
        key_list.remove(a_key)
        if len(del_container) > 0:
            print("删除已处理的元素")
            for item in del_container:
                key_list.remove(item)



# a_word = "腋下长了一个小疙瘩是什么东西"
# b_word = "什么东西吃蟑螂(四个字)"
# cos_val = cal_cos(a_word, b_word)
# print(cos_val)
# print("的余弦值：%f " % ( cos_val))
# print(cut_word(b_word))

# 1. 使用缓存中的分词
# 2. 余弦值超过0.8的视为一组