import codecs
from bs4 import BeautifulSoup
from konlpy.tag import Okt
from gensim.models import word2vec

text = codecs.open("sent.content.txt", "r", encoding = "utf-8")

f = open("sent.content.txt", "r", encoding="utf-8")
print(f.read(20))

word_dict = {}
lines = f.read().split("\n")
print(lines[10])

morph_analyzer = Okt()
morph_analyzer1 = Okt()
for line in lines:
    morph_analysed = morph_analyzer.pos(line)
    for word in morph_analysed:
        if word[1] == "Noun":
            if not (word[0] in word_dict):
                word_dict[word[0]] = 0
            word_dict[word[0]] += 1    

keys = sorted(word_dict.items(), key=lambda x:x[1], reverse=True)

print(keys[:10])

for word, count in keys[:20]:
    print("{0}{1}".format(word, count))
    
f.close()

with open("sent.content.txt", "r", encoding="utf-8") as fr:
    results = []
    lines = fr.read().split("\n")
    for line in lines:
        morph_analysed = morph_analyzer.pos(line, norm=True, stem=True) # stem=True로 어근 출력
        r = []
        for word in morph_analysed:
            if not word[1] in ["Josa", "Punctuation", "Foreign", "Suffix", "Eomi"]:
                r.append(word[0])
            # 결과에서 제외 할 품사 입력하기
        #print(r) 
        rl = (" ".join(r)).strip() #공백제거
        results.append(rl)        

print(results[:4])

data_file = "sent.content.txt"
with open(data_file, "w", encoding="utf-8") as fw:
    fw.write("\n".join(results))
           
from gensim.models import word2vec
data = word2vec.LineSentence(data_file)
print(data)
model = word2vec.Word2Vec(data, size=100, window=10, hs=1, min_count=2, sg=1)
# CBOW, Skip-gram(0)
model.init_sims(replace=True) #필요없는 메모리는 unload
#model.save("senti_cat.model")
print("done")

##model = word2vec.Word2Vec.load("senti_cat.model")
##print(model.similarity("기쁘다", "슬프다")) 
##print(model.similarity("기쁘다", "행복하다")) 

print(model.most_similar("기쁘다"))

from gensim.models import KeyedVectors
model.wv.save_word2vec_format("senti_cat_model")

# PCA 돌리기 위해 모델 텐서를 저장한다
#!python -m gensim.scripts.word2vec2tensor --input gimal_model --output gimal_model

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

data=pd.read_excel('./senti_cat.xlsx')
pd.set_option('display.max_rows',500)
data

v=data['기준동사분류'].value_counts()
v2=v.to_frame()
#v2.to_excel('senti_cat_v2.xlsx')
v2

size1=data.groupby(['기준동사분류','관련동사분류']).size()
size2=size1.to_frame()
size2