@article {10.3844/jcssp.2022.18.25,
article_type = {journal},
title = {Topic-Transformer for Document-Level Language Understanding},
author = {Hourrane, Oumaima and Benlahmar, El Habib},
volume = {18},
number = {1},
year = {2022},
month = {Jan},
pages = {18-25},
doi = {10.3844/jcssp.2022.18.25},
url = {https://thescipub.com/abstract/jcssp.2022.18.25},
abstract = {As long as natural language processing applications are considered prediction problems with insufficient context, usually referred to as a single sentence or paragraph, this does not reveal how humans perceive natural language. When reading a text, humans are sensitive to much more context, such as the rest or other relevant documents. This study focuses on simultaneously capturing syntax and global semantics from a text, thus acquiring document-level understanding. Accordingly, we introduce a Topic-Transformer that combines the benefits of a neural topic model that captures global semantic information and a transformer-based language model, which can capture the local structure of texts both semantically and syntactically. Experiments on various datasets confirm that our model has a lower perplexity metric compared to standard transformer architecture and the recent topic-guided language models and generates topics that are conceivably coherent compared to those of regular Latent Dirichlet Allocation (LDA) topic model.},
journal = {Journal of Computer Science},
publisher = {Science Publications}
}