deep-atrous-ner/preprocess.py at master · randomrandom/deep-atrous-ner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
from data.conll_loader import ConllLoader
from data.datasets.conll_2003 import preprocess_conll

__author__ = 'georgi.val.stoyan0v@gmail.com'

def preprocess_files():
    preprocess_conll.preprocess_file('./data/datasets/conll_2003/', 'eng.testa', ConllLoader.TSV_DELIM,
                                     ConllLoader.DEFAULT_VOCABULARY_SIZE, ConllLoader.DEFAULT_MAX_DATA_LENGTH)

    preprocess_conll.preprocess_file('./data/datasets/conll_2003/', 'eng.testb', ConllLoader.TSV_DELIM,
                                     ConllLoader.DEFAULT_VOCABULARY_SIZE, ConllLoader.DEFAULT_MAX_DATA_LENGTH)

    preprocess_conll.preprocess_file('./data/datasets/conll_2003/', 'eng.train', ConllLoader.TSV_DELIM,
                                     ConllLoader.DEFAULT_VOCABULARY_SIZE, ConllLoader.DEFAULT_MAX_DATA_LENGTH)

preprocess_files()