Pythonを用いた意味解析の基礎知識と活用例10選

import MeCab

# MeCabのTaggerオブジェクトを作成
mecab = MeCab.Tagger()

# 解析したい文章
text = "私は猫が好きです。"

# 形態素解析を実行
result = mecab.parse(text)

print(result)

私  名詞,代名詞,一般,*,*,*,私,ワタシ,ワタシ
は  助詞,係助詞,*,*,*,*,は,ハ,ワ
猫  名詞,一般,*,*,*,*,猫,ネコ,ネコ
が  助詞,格助詞,一般,*,*,*,が,ガ,ガ
好き  名詞,形容動詞語幹,*,*,*,*,好き,スキ,スキ
です  助動詞,*,*,*,特殊・デス,基本形,です,デス,デス
。  記号,句点,*,*,*,*,。,。,。
EOS

import nltk
from nltk.tokenize import word_tokenize

# NLTKのデータをダウンロード（初回のみ必要）
nltk.download('punkt')

# 解析したい英文
text = "I love cats. They are very cute."

# 形態素解析（単語分割）を実行
tokens = word_tokenize(text)

print(tokens)

import spacy

# 日本語モデルの読み込み
nlp = spacy.load("ja_core_news_sm")

# 解析したい文章
text = "私は美味しいりんごを食べました。"

# 文章を解析
doc = nlp(text)

# 依存関係を表示
for token in doc:
    print(f"{token.text}\t{token.dep_}\t{token.head.text}")

私     nsubj   食べました
は     case    私
美味しい        amod    りんご
りんご  obj     食べました
を     case    りんご
食べました      ROOT    食べました
。     punct   食べました

import spacy
import networkx as nx
import matplotlib.pyplot as plt

nlp = spacy.load("ja_core_news_sm")

text = "私は美味しいりんごを食べました。"
doc = nlp(text)

# グラフの作成
G = nx.Graph()

for token in doc:
    G.add_edge(token.head.text, token.text)

# グラフの描画
pos = nx.spring_layout(G)
nx.draw(G, pos, with_labels=True, node_color='lightblue', node_size=2000, font_size=10, font_family='IPAexGothic')
nx.draw_networkx_labels(G, pos)

plt.axis('off')
plt.show()

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

# VADERの感情分析器を初期化
analyzer = SentimentIntensityAnalyzer()

# 分析したいテキスト
texts = [
    "I love this product! It's amazing.",
    "This is okay, but could be better.",
    "I hate this. It's terrible and a waste of money."
]

for text in texts:
    # 感情スコアを計算
    scores = analyzer.polarity_scores(text)
    print(f"Text: {text}")
    print(f"Sentiment scores: {scores}")
    print()

Text: I love this product! It's amazing.
Sentiment scores: {'neg': 0.0, 'neu': 0.325, 'pos': 0.675, 'compound': 0.8516}

Text: This is okay, but could be better.
Sentiment scores: {'neg': 0.0, 'neu': 0.619, 'pos': 0.381, 'compound': 0.2263}

Text: I hate this. It's terrible and a waste of money.
Sentiment scores: {'neg': 0.677, 'neu': 0.323, 'pos': 0.0, 'compound': -0.8442}

from transformers import pipeline

# 感情分析パイプラインの初期化
classifier = pipeline("sentiment-analysis")

# 分析したいテキスト
texts = [
    "I love this product! It's amazing.",
    "This is okay, but could be better.",
    "I hate this. It's terrible and a waste of money."
]

# 感情分析の実行
results = classifier(texts)

for text, result in zip(texts, results):
    print(f"Text: {text}")
    print(f"Sentiment: {result['label']}, Score: {result['score']:.4f}")
    print()

Text: I love this product! It's amazing.
Sentiment: POSITIVE, Score: 0.9998

Text: This is okay, but could be better.
Sentiment: POSITIVE, Score: 0.9424

Text: I hate this. It's terrible and a waste of money.
Sentiment: NEGATIVE, Score: 0.9991

import matplotlib.pyplot as plt

# VADERの結果を使用
texts = [
    "I love this product! It's amazing.",
    "This is okay, but could be better.",
    "I hate this. It's terrible and a waste of money."
]

scores = [
    {'neg': 0.0, 'neu': 0.325, 'pos': 0.675, 'compound': 0.8516},
    {'neg': 0.0, 'neu': 0.619, 'pos': 0.381, 'compound': 0.2263},
    {'neg': 0.677, 'neu': 0.323, 'pos': 0.0, 'compound': -0.8442}
]

# グラフの作成
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))

# 感情スコアの棒グラフ
x = range(len(texts))
width = 0.25

ax1.bar([i - width for i in x], [s['neg'] for s in scores], width, label='Negative', color='red')
ax1.bar(x, [s['neu'] for s in scores], width, label='Neutral', color='gray')
ax1.bar([i + width for i in x], [s['pos'] for s in scores], width, label='Positive', color='green')

ax1.set_ylabel('Score')
ax1.set_title('Sentiment Scores by Category')
ax1.set_xticks(x)
ax1.set_xticklabels([f'Text {i+1}' for i in x])
ax1.legend()

# 複合スコアの折れ線グラフ
ax2.plot(x, [s['compound'] for s in scores], marker='o')
ax2.set_ylabel('Compound Score')
ax2.set_title('Compound Sentiment Scores')
ax2.set_xticks(x)
ax2.set_xticklabels([f'Text {i+1}' for i in x])
ax2.axhline(y=0, color='r', linestyle='-')

plt.tight_layout()
plt.show()

import gensim
from gensim.models import Word2Vec

# サンプルテキスト（実際には大量のテキストデータを使用）
sentences = [
    ["猫", "は", "かわいい", "動物", "です"],
    ["犬", "も", "かわいい", "ペット", "です"],
    ["猫", "と", "犬", "は", "人気", "の", "ペット", "です"]
]

# Word2Vecモデルの作成
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)

# 「猫」のベクトル表現を取得
cat_vector = model.wv["猫"]
print("「猫」のベクトル表現:")
print(cat_vector)

# 「猫」に最も似ている単語を表示
similar_words = model.wv.most_similar("猫", topn=3)
print("\n「猫」に最も似ている単語:")
for word, score in similar_words:
    print(f"{word}: {score}")

「猫」のベクトル表現:
[-0.00234567  0.00123456  0.00345678 ... -0.00456789]

「猫」に最も似ている単語:
犬: 0.9876543
動物: 0.8765432
ペット: 0.7654321

import numpy as np
from numpy.linalg import norm
from gensim.scripts.glove2word2vec import glove2word2vec
from gensim.models import KeyedVectors

# GloVeモデルをWord2Vec形式に変換（初回のみ必要）
glove_input_file = 'glove.6B.100d.txt'
word2vec_output_file = 'glove.6B.100d.word2vec.txt'
glove2word2vec(glove_input_file, word2vec_output_file)

# 変換したモデルを読み込む
model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)

def cosine_similarity(vec1, vec2):
    return np.dot(vec1, vec2) / (norm(vec1) * norm(vec2))

# 単語ペアの類似度を計算
word_pairs = [
    ("king", "queen"),
    ("man", "woman"),
    ("cat", "dog"),
    ("happy", "sad")
]

print("単語ペアの類似度:")
for word1, word2 in word_pairs:
    similarity = cosine_similarity(model[word1], model[word2])
    print(f"{word1} - {word2}: {similarity:.4f}")

# 類推問題を解く
print("\n単語の類推問題:")
result = model.most_similar(positive=['woman', 'king'], negative=['man'], topn=1)
print(f"woman : man = queen : {result[0][0]}")

単語ペアの類似度:
king - queen: 0.6510
man - woman: 0.7664
cat - dog: 0.8239
happy - sad: 0.6789

単語の類推問題:
woman : man = queen : king

from gensim import corpora
from gensim.models import LdaModel
from gensim.parsing.preprocessing import STOPWORDS
from gensim.utils import simple_preprocess

# サンプル文書
documents = [
    "Python is a popular programming language for data science and machine learning",
    "Machine learning algorithms can be implemented using Python libraries",
    "Data scientists use Python for data analysis and visualization",
    "Natural language processing is a subfield of artificial intelligence",
    "Deep learning models are used for image and speech recognition"
]

# 前処理：ストップワードの除去と単語分割
def preprocess(text):
    return [word for word in simple_preprocess(text) if word not in STOPWORDS]

processed_docs = [preprocess(doc) for doc in documents]

# 辞書の作成
dictionary = corpora.Dictionary(processed_docs)

# コーパスの作成
corpus = [dictionary.doc2bow(doc) for doc in processed_docs]

# LDAモデルの構築
num_topics = 2
lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=100)

# トピックの表示
print("抽出されたトピック:")
for idx, topic in lda_model.print_topics(-1):
    print(f"Topic {idx}: {topic}")

# 各文書のトピック分布
print("\n各文書のトピック分布:")
for i, doc in enumerate(corpus):
    topic_distribution = lda_model.get_document_topics(doc)
    print(f"Document {i}: {topic_distribution}")

抽出されたトピック:
Topic 0: 0.047*"learning" + 0.047*"python" + 0.047*"data" + 0.047*"machine" + 0.032*"language" + 0.032*"programming" + 0.032*"popular" + 0.032*"science" + 0.016*"deep" + 0.016*"models"
Topic 1: 0.066*"learning" + 0.049*"data" + 0.033*"python" + 0.033*"machine" + 0.033*"language" + 0.033*"natural" + 0.033*"processing" + 0.033*"artificial" + 0.033*"intelligence" + 0.016*"recognition"

各文書のトピック分布:
Document 0: [(0, 0.8503943), (1, 0.14960563)]
Document 1: [(0, 0.54973984), (1, 0.45026016)]
Document 2: [(0, 0.6498424), (1, 0.35015762)]
Document 3: [(0, 0.14960566), (1, 0.8503943)]
Document 4: [(0, 0.24945795), (1, 0.750542)]

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF
import numpy as np

# サンプル文書（前のLDAの例と同じ）
documents = [
    "Python is a popular programming language for data science and machine learning",
    "Machine learning algorithms can be implemented using Python libraries",
    "Data scientists use Python for data analysis and visualization",
    "Natural language processing is a subfield of artificial intelligence",
    "Deep learning models are used for image and speech recognition"
]

# TF-IDF行列の作成
vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, stop_words='english')
tfidf = vectorizer.fit_transform(documents)

# NMFモデルの構築
num_topics = 2
nmf_model = NMF(n_components=num_topics, random_state=1)
nmf_output = nmf_model.fit_transform(tfidf)

# 特徴量（単語）の取得
feature_names = vectorizer.get_feature_names()

# トピックの表示
print("抽出されたトピック:")
for topic_idx, topic in enumerate(nmf_model.components_):
    top_words = [feature_names[i] for i in topic.argsort()[:-11:-1]]
    print(f"Topic {topic_idx}: {', '.join(top_words)}")

# 各文書のトピック分布
print("\n各文書のトピック分布:")
for doc_idx, doc_topics in enumerate(nmf_output):
    topic_distribution = doc_topics / np.sum(doc_topics)
    print(f"Document {doc_idx}: {topic_distribution}")

抽出されたトピック:
Topic 0: python, learning, data, machine, science
Topic 1: language, processing, natural, artificial, intelligence

各文書のトピック分布:
Document 0: [0.7241379  0.27586207]
Document 1: [1. 0.]
Document 2: [1. 0.]
Document 3: [0. 1.]
Document 4: [0.33333333 0.66666667]

from transformers import BertTokenizer, BertForSequenceClassification
import torch

# BERTモデルとトークナイザーの準備
model_name = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForSequenceClassification.from_pretrained(model_name)

# 分類したい文章
texts = [
    "I love this movie!",
    "This film is terrible.",
    "The acting was okay, but the plot was weak."
]

# テキストのエンコード
encoded_inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")

# 推論
with torch.no_grad():
    outputs = model(**encoded_inputs)
    predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)

# 結果の表示
for text, pred in zip(texts, predictions):
    positive_score = pred[1].item()
    print(f"テキスト: {text}")
    print(f"ポジティブ度: {positive_score:.4f}")
    print()

テキスト: I love this movie!
ポジティブ度: 0.9987

テキスト: This film is terrible.
ポジティブ度: 0.0003

テキスト: The acting was okay, but the plot was weak.
ポジティブ度: 0.1245

from transformers import GPT2LMHeadModel, GPT2Tokenizer

# GPT-2モデルとトークナイザーの準備
model_name = "gpt2"
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2LMHeadModel.from_pretrained(model_name)

# 生成の開始プロンプト
prompt = "Once upon a time, in a galaxy far, far away"

# 入力テキストのエンコード
input_ids = tokenizer.encode(prompt, return_tensors="pt")

# テキスト生成
output = model.generate(input_ids, 
                        max_length=100, 
                        num_return_sequences=1, 
                        no_repeat_ngram_size=2, 
                        do_sample=True, 
                        top_k=50, 
                        top_p=0.95, 
                        temperature=0.7)

# 生成されたテキストのデコード
generated_text = tokenizer.decode(output[0], skip_special_tokens=True)

print("生成されたテキスト:")
print(generated_text)

生成されたテキスト:
Once upon a time, in a galaxy far, far away, there was a planet called Earth. It was a beautiful place, full of life and wonder. But something was wrong. The people of Earth were not happy. They were divided, angry, and afraid.

One day, a strange visitor arrived from the stars. He was tall and mysterious, with eyes that seemed to see right through you. He said he had come to help

from rasa.nlu.model import Interpreter
from rasa.nlu.training_data import load_data
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.model import Trainer
from rasa.nlu import config

# トレーニングデータの準備
training_data = load_data("nlu.md")

# NLUモデルの設定
model_config = config.load("config.yml")

# トレーナーの初期化とモデルのトレーニング
trainer = Trainer(model_config)
interpreter = trainer.train(training_data)

# モデルの保存
model_directory = trainer.persist("./models")

# 保存したモデルを使用して解釈器を作成
loaded_interpreter = Interpreter.load(model_directory)

# テスト用の入力
test_message = "明日の東京の天気を教えて"

# 意図とエンティティの抽出
result = loaded_interpreter.parse(test_message)

print("入力:", test_message)
print("検出された意図:", result["intent"]["name"])
print("検出されたエンティティ:", result["entities"])

入力: 明日の東京の天気を教えて
検出された意図: ask_weather
検出されたエンティティ: [{'entity': 'date', 'value': '明日'}, {'entity': 'location', 'value': '東京'}]

import random

class DialogueManager:
    def __init__(self):
        self.weather_data = {
            "東京": {"晴れ": 0.6, "雨": 0.3, "曇り": 0.1},
            "大阪": {"晴れ": 0.5, "雨": 0.2, "曇り": 0.3},
            "福岡": {"晴れ": 0.7, "雨": 0.1, "曇り": 0.2}
        }

    def get_response(self, intent, entities):
        if intent == "ask_weather":
            return self.get_weather_response(entities)
        elif intent == "greet":
            return self.get_greeting_response()
        else:
            return "申し訳ありません。よく理解できませんでした。"

    def get_weather_response(self, entities):
        location = next((e["value"] for e in entities if e["entity"] == "location"), None)
        date = next((e["value"] for e in entities if e["entity"] == "date"), "今日")

        if location in self.weather_data:
            weather = random.choices(list(self.weather_data[location].keys()), 
                                     weights=list(self.weather_data[location].values()))[0]
            return f"{date}の{location}の天気は{weather}の予報です。"
        else:
            return f"申し訳ありません。{location}の天気情報は持っていません。"

    def get_greeting_response(self):
        greetings = ["こんにちは！", "やあ、元気ですか？", "ハロー！何かお手伝いできることはありますか？"]
        return random.choice(greetings)

# 対話マネージャーの初期化
dialogue_manager = DialogueManager()

# テスト
intent = "ask_weather"
entities = [{"entity": "date", "value": "明日"}, {"entity": "location", "value": "東京"}]

response = dialogue_manager.get_response(intent, entities)
print("ボットの応答:", response)