I have a list of strings (1.5 million) where list of strings like
['zzh2z24nV5Rl5TMKpSZFGBINFUVq', 'zzDD78WbbuiJmuu39V0opHMzArTU',
'zz+GR08MrX9sDVH14wK0ql3z7Hh22+mj2IhnxO/69b0=',
'zz+GR08MrX9sDVH14wK0ql3z7Hh22+mj2IhnxO/69b0=',
'zytUgOn10HEL2P1nt0JN', 'zytUgOn10HEL2P1nt0JN',
'zxwQJmJQp0MILZt1vKyhnSg65RgF', 'zxwQJmJQp0MILZt1vKyhnSg65RgF',
'zxnJy0uha0iIZdRxS6GhA%2BSxpvQdqiguF8fws11Xqcw%3D',
'zxfea5z0riInF4qMqkXLoZv96k2a', 'zxfea5z0riInF4qMqkXLoZv96k2a',
'zxJSM5pcPRN8YTz/gm5mf2Y61M3A26biLsUMKlu20OE=',
'zwgOkuH7AmkDxOUz3FD7xFAkTgvCBd46IVTOsEXZxOM%3D',
'zvvxBkk9qVyxvMrqZ3xC9aOE9ufKIt6jNbxhUphKkow%3D',
'zvYYj1FYNsX5EBN8mS+fhTi5bNcJrdp+KnJPf9vG1cg=', ...]
I want to plot graph of most similar words like this . At least 1-2 thousand. And also I need a list of similar vectors/words. What I do:
I am considering strings as sentences so any symbol in the string is a word.
Am choosing unique strings and transform them into sentences using split like
list_of_sentences = [['2', '2', '1', '1', '1', '%', '7', 'C', '8', '6', '2', '1', '9', '5', '9', '%', '7', 'C', '9', 'b', '0', '5', '8', '5', '5', '2', 'd', '0', '8', 'e', '8', '9', 'e', '9'], [...], ... ]
Convert words into vectors
model = Word2Vec(list_of_sentences[:1000]), min_count=2, size=50, workers=4)
Extract vocabulary of word-vectors from model and collapse to 2 dimension, plot grap
def tsne_plot(model):
labels = []
tokens = []
for word in model.wv.vocab:
tokens.append(model[word])
labels.append(word)
tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
new_values = tsne_model.fit_transform(tokens)
x = []
y = []
for value in new_values:
x.append(value[0])
y.append(value[1])
plt.figure(figsize=(10, 10))
for i in range(len(x)):
plt.scatter(x[i],y[i])
plt.annotate(labels[i],
xy=(x[i], y[i]),
xytext=(5, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.show()
tsne_plot(model)
But I am getting a graph of the similarity of characters, not strings of symbol similarity
What I do wrong and how I can get list of similar strings?