money-mining-python/key_index.py

# -*- coding:utf-8 -*-

import config
import tools
import mmap


def main():
    
    # 关键词索引容器,
    key_index = []

    with open(config.KEY_FILE, "r", encoding=config.ENCODING_CHARSET) as fkey, \
        mmap.mmap(fkey.fileno(), 0, access=mmap.ACCESS_READ) as fmmap:
        
        # 总大小
        total_num = fmmap.size()

        while True:
            # 读取光标位置
            cur_pos = fmmap.tell()
            # 把光标移动到下一行
            line = fmmap.readline()
            # 如果没有数据则结束
            if not line :
                break

            # 建立关键词序号和位置的关系，以索引当行号（0基）
            key_index.append(cur_pos)

            # 进度显示
            tools.tip_in_size(total_num, cur_pos)
        
    with open("./data/tmp/key_index_test.csv", "w", encoding=config.ENCODING_CHARSET) as f:
        f.write(",".join([str(i) for i in key_index]))

        
    # 保存索引
    # tools.save_obj(config.KEY_INDEX_CACHE, key_index)


if __name__ == '__main__':

    TITLE = "关键词索引"

    # 日志配置初始化
    tools.init_log()
    tools.log_start_msg(TITLE)

    main()
    
    tools.log_end_msg(TITLE)