テキスト分類 (PyTorch)

# !apt install aptitude
# !aptitude install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y
# !pip install mecab-python3==0.7
      

import MeCab
mecab = MeCab.Tagger('-Ochasen')
      

res = mecab.parse('こんにちは、私はキカガクです。')
      

print(res)
      

res
      

text1 = 'キカガクでは、ディープラーニングを含んだ機械学習や人工知能の教育を行っています。'
text2 = '代表の吉崎は大学院では機械学習・ロボットのシステム制御、画像処理の研究に携わっていました。'
text3 = '機械学習、システム制御、画像処理ではすべて線形代数とプログラミングが不可欠になります。'
      

# 形態素解析
res = mecab.parse(text1)
print(res)
      

res
      

# 改行ごとに分割し、リストに格納
res.split('\n') 
      

# 最後の 2 つの要素までをスライス
res.split('\n')[:-2]
      

# 1 つ目の単語をスライス
res.split('\n')[0]
      

# 品詞の取得
res .split('\n')[0].split('\t')[3]
      

nouns = [] # 品詞が名詞 (noun) である単語を格納するリスト
res = mecab.parse(text1)
words = res.split('\n')[:-2]
for word in words:
    part = word.split('\t')
    if '名詞' in part[3]:
        nouns.append(part[0])
      

nouns
      

def get_nouns(text):
    nouns = []
    res = mecab.parse(text)
    words = res.split('\n')[:-2]
    for word in words:
        part = word.split('\t')
        if '名詞' in part[3]:
            nouns.append(part[0])
    return nouns
      

nouns1 = get_nouns(text1)
nouns1
      

nouns2 = get_nouns(text2)
nouns2
      

nouns3 = get_nouns(text3)
nouns3
      

nouns1
      

# 要素を半角スペースで結合
' '.join(nouns1)
      

nouns_list = [nouns1, nouns2, nouns3]
corpus = []
for nouns in nouns_list:
  corpus.append(' '.join(nouns))
      

corpus
      

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
x = vectorizer.fit_transform(corpus)
      

vectorizer.vocabulary_
      

x = x.toarray()
print(x)
      

from google.colab import files
uploaded = files.upload()
      

!unzip -d text texts.zip
      

# 解凍したファイルの確認
!ls text/
      

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
      

from glob import glob
      

# ディレクトリの取得
directories = glob('/content/text/*')
directories
      

filepaths = glob('{}/*.txt'.format(directories[0]))
filepaths[:3]
      

with open(filepaths[0], encoding='utf-8') as f:
  text = ''.join(f.readlines()[2:])
print(text)
      

for (i, directory) in enumerate(directories):
    print(i, directory)
    print('- - -')
      

texts, labels = [], []
for (i, directory) in enumerate(directories):
    #各ディレクトリ内のtxtファイルのパスをすべて取得
    filepaths = glob('{}/*.txt'.format(directory))
    # テキストを読み込んで、内容をtextに格納、ラベルも併せて格納
    for filepath in filepaths:
        with open(filepath, encoding='utf-8') as f:
            text = ''.join(f.readlines()[2:])  # URL等の先頭２行を除いた各行の文章を連結（join）して格納
            texts.append(text)
            labels.append(i)
      

len(texts), len(labels)
      

texts[0]
      

labels[0]
      

import MeCab
mecab = MeCab.Tagger('-Ochasen')
      

def get_nouns(text):
    nouns = []
    res = mecab.parse(text)
    words = res.split('\n')[:-2]
    for word in words:
        part = word.split('\t')
        if '名詞' in part[3]:
            nouns.append(part[0])
    return nouns
      

word_collect = []
for text in texts:
    nouns = get_nouns(text)
    word_collect.append(' '.join(nouns))
      

word_collect[0]
      

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(min_df=20)
x = vectorizer.fit_transform(word_collect)
x = x.toarray()
      

len(vectorizer.vocabulary_)
      

len(x)
      

import torch
      

# tensor形式へ変換
x = torch.tensor(x, dtype=torch.float32)
t = torch.tensor(labels, dtype=torch.int64)
      

x.shape,t.shape
      

type(x), x.dtype
      

type(t), t.dtype
      

import torch.utils.data
      

dataset = torch.utils.data.TensorDataset(x, t)
dataset
      

# 全サンプル数
len(dataset)
      

n_train = int(len(dataset) * 0.6)
n_val = int(len(dataset) * 0.2)
n_test = len(dataset) - n_train - n_val
      

# データの分割
torch.manual_seed(0)

train, val, test = torch.utils.data.random_split(dataset, [n_train, n_val, n_test])
      

len(train), len(val), len(test)
      

!pip install pytorch_lightning
      

import torch.nn as nn
import torch.nn.functional as F

import pytorch_lightning as pl
from pytorch_lightning import Trainer
      

class TrainNet(pl.LightningModule):

    @pl.data_loader
    def train_dataloader(self):
        return torch.utils.data.DataLoader(train, self.batch_size, shuffle=True, num_workers=self.num_workers)

    def training_step(self, batch, batch_nb):
        x, t = batch
        y = self.forward(x)
        loss = self.lossfun(y, t)
        results = {'loss': loss}
        return results
      

class ValidationNet(pl.LightningModule):

    @pl.data_loader
    def val_dataloader(self):
        return torch.utils.data.DataLoader(val, self.batch_size, num_workers=self.num_workers)

    def validation_step(self, batch, batch_nb):
        x, t = batch
        y = self.forward(x)
        loss = self.lossfun(y, t)
        y_label = torch.argmax(y, dim=1)
        acc = torch.sum(t == y_label) * 1.0 / len(t)
        results = {'val_loss': loss, 'val_acc': acc}
        return results

    def validation_end(self, outputs):
        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['val_acc'] for x in outputs]).mean()
        results = {'val_loss': avg_loss, 'val_acc': avg_acc}
        return results
      

class TestNet(pl.LightningModule):

    @pl.data_loader
    def test_dataloader(self):
        return torch.utils.data.DataLoader(test, self.batch_size, num_workers=self.num_workers)

    def test_step(self, batch, batch_nb):
        x, t = batch
        y = self.forward(x)
        loss = self.lossfun(y, t)
        y_label = torch.argmax(y, dim=1)
        acc = torch.sum(t == y_label) * 1.0 / len(t)
        results = {'test_loss': loss, 'test_acc': acc}
        return results

    def test_end(self, outputs):
        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['test_acc'] for x in outputs]).mean()
        results = {'test_loss': avg_loss, 'test_acc': avg_acc}
        return results
      

class Net(TrainNet, ValidationNet, TestNet):

    def __init__(self, batch_size=128, num_workers=8):
        super(Net, self).__init__()
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.fc1 = nn.Linear(5795, 200)
        self.fc2 = nn.Linear(200, 9)

    def lossfun(self, y, t):
        return F.cross_entropy(y, t)

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.01)

    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        return x
      

# cuDNN に対する再現性の確保
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
      

# 乱数のシードを固定
torch.manual_seed(0)

net = Net(batch_size=128)
trainer = Trainer(gpus=1, max_nb_epochs=20)

# モデルの学習
trainer.fit(net)
      

# 最終的なエポックの検証データに対する結果
trainer.callback_metrics
      

# テストデータに対する検証
trainer.test()

# テストデータに対する結果
trainer.callback_metrics