
import pandas as pd
import re
from datetime import datetime
import string
from nltk.tokenize import word_tokenize
import nltk
import matplotlib.pyplot as plt
nltk.download('punkt')
nltk.download('stopwords')
stopwords = nltk.corpus.stopwords.words('portuguese')

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import FunctionTransformer
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, matthews_corrcoef, accuracy_score
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsRestClassifier

[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.


dados = pd.read_csv('/content/drive/MyDrive/DATA/pedidos-recursos-features-process.csv')


def clear_values_regex(df):
    df = df.replace(r'\r\n\r\n',' ', regex=True)
    df = df.replace(r'\r\n',' ', regex=True) 
    df = df.replace(r'\r \r',' ', regex=True)
    return df


dados = clear_values_regex(dados)
dados['conteudo_resposta'] = dados['conteudo_resposta'].fillna('')
dados = dados[dados['atendimento_resposta'] != 'Não Classificado']


def limpando_texto(mensagem): 
  # Diminuindo os caracteres
    mensagem = mensagem.lower()
  # url
    regex_url = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
    mensagem = re.sub(regex_url,'site',mensagem)
  # data replace
    mensagem = re.sub(r'\d{2}.\d{2}.\d{4}','data',mensagem)
  # remove pontuação
    table    = str.maketrans('', '', string.punctuation)
    stripped = [palavra.translate(table) for palavra in mensagem.split()]
    mensagem  = ' '.join(stripped).strip()
  # remove stopword
    mensagem_tokens = word_tokenize(mensagem)
    mensagem = [word for word in mensagem_tokens if not word in stopwords]
    mensagem = " ".join(mensagem)
    return mensagem


dados['conteudo_pedido'] = dados['conteudo_pedido'].apply(limpando_texto)
dados['conteudo_resposta'] = dados['conteudo_resposta'].apply(limpando_texto)


def get_text_data(x):
    return [record[0] for record in x]


def get_text_data_2(x):
    return [record[1] for record in x]

transformer_text = FunctionTransformer(get_text_data)
transformer_text_2 = FunctionTransformer(get_text_data_2)


def train_models(X_train,y_train):
  pipeline = Pipeline([
      ('features', FeatureUnion([
              ('text_features', Pipeline([
                  ('selector', transformer_text),
                  ('vec', TfidfVectorizer(analyzer='word'))
              ])),
                ('text_features2', Pipeline([
                  ('selector', transformer_text_2),
                  ('vec', TfidfVectorizer(analyzer='word'))
              ]))
          ])),
      ('svc', OneVsRestClassifier(SVC(C = 100 , gamma=0.001 , kernel='rbf')))
  ])
  pipeline.fit(X_train,y_train)
  return pipeline


def label_encode(x):
    if x == 'Atendido':
        return 2
    elif x == 'Parcialmente Atendido':
        return 1
    else:
        return 0

def split_data(df):
  labels = df['atendimento_resposta'].apply(label_encode)
  df = df.drop(columns=['atendimento_resposta'])
  X_train, X_test, y_train , y_test = train_test_split(df, labels, test_size=0.2, random_state=5)
  X_train = X_train.to_numpy()
  X_test = X_test.to_numpy()
  return X_train, X_test, y_train , y_test


def test(df):
  X_train, X_test, y_train , y_test = split_data(df)
  model = train_models(X_train,y_train)
  predicted_text = model.predict(X_test)
  accuracy = accuracy_score(y_test, predicted_text)
  f1 = f1_score(y_test, predicted_text,average='micro')
  mcc = matthews_corrcoef(y_test, predicted_text)
  result = pd.DataFrame(zip([float('{:04.2f}'.format(accuracy*100))],[float('{:04.2f}'.format(f1*100))],[mcc]),columns=['Acurácia','F1','MCC'])
  return result


dados = dados[['conteudo_pedido','conteudo_resposta','atendimento_resposta']]


resultados = test(dados)
resultados

Limpeza de Texto¶

Modelagem¶