-1

I have a dataset with Russian text, which looks like this: enter image description here

I am trying to pre-process this dataset and split it to train,dev and testing datasets by using the following code:

# coding=utf-8
import os
import argparse
import xml.etree.ElementTree as ET
import random
import math
from collections import Counter
from utils import semeval2014term_to_aspectsentiment_hr
from copy import copy, deepcopy

parser = argparse.ArgumentParser(description='Generate finetuning corpus for restaurants.')

parser.add_argument('--noconfl',
                    action='store_true',
                    default=False,
                    help='Remove conflicting sentiments from labels')

parser.add_argument('--istrain',
                    action='store_true',
                    default=False,
                    help='If is a training set we split of 10% and output train_full, train_split, dev. Default is testset creating no split')

parser.add_argument("--files",
                    type=str,
                    nargs='+',
                    action="store",
                    help="File that contains the data used for training. Multiple paths will mix the datasets.")

parser.add_argument("--output_dir",
                    type=str,
                    action="store",
                    default="data/transformed/untitled",
                    help="output dir of the dataset(s)")

parser.add_argument("--upsample",
                    type=str,
                    action="store",
                    default=None,
                    help="please add a string with 3 numbers like '0.5 0.3 0.2' representing relative numbers of 'POS NEG NEU' adding to 1"
                         " which represents target distribution - only valid in non-confl case")

parser.add_argument("--seed",
                    type=int,
                    action="store",
                    default=41,
                    help="random seed, effects on upsampling and validationset")

args = parser.parse_args()


# 1. Load The Dataset
# 2. Create Bert-Pair Style Format
# 3. Save Train, Validation and so on

def split_shuffle_array(ratio, array, rseed):
    # split_ratio_restaurant = .076  # for 150 sentence in conflicting case
    # split_ratio_laptops = .101  # for 150 sentences in conflicting case
    random.Random(rseed).shuffle(array)
    m = math.floor(ratio * len(array))
    return array[0:m], array[m::]


def create_sentence_pairs(sents, aspect_term_sentiments):
    # create sentence_pairs

    all_sentiments = []
    sentence_pairs = []
    labels = []

    for ix, ats in enumerate(aspect_term_sentiments):
        s = sents[ix]
        for k, v in ats:
            all_sentiments.append(v)
            sentence_pairs.append((s, k))
            labels.append(v)
    counts = Counter(all_sentiments)

    return sentence_pairs, labels, counts

def upsample_data(sentence_pairs, labels, target_ratios={'POS': 0.53, 'NEG': 0.21, 'NEU': 0.26}):
    # one question: should we upsample sentencepairs, where the sentence only occurs once?!
    print('Upsampling data ...')
    # print(sentence_pairs, labels)  # is list of pairs -> decide which pair to upsample ...

    # 0. compute indeex subsets for every example
    # 1. compute how many samples to sample ->

    ix_subsets = {
        'POS': [],
        'NEG': [],
        'NEU': []
    }
    ratios_subsets = {
        'POS': 0,
        'NEG': 0,
        'NEU': 0
    }
    examples_to_add = {
        'POS': 0,
        'NEG': 0,
        'NEU': 0
    }
    n = float(len(labels))
    for ix, l in enumerate(labels):
        ix_subsets[l].append(ix)
        ratios_subsets[l] += (1.0 / n)

    t_keys = target_ratios.keys()
    tmp = [math.floor(target_ratios[k] * n) - len(ix_subsets[k]) for k in t_keys]
    class_nothing_to_add = list(t_keys)[tmp.index(min(tmp))]
    print(t_keys)
    print(ratios_subsets)
    print(tmp)
    print(class_nothing_to_add)
    # print(ix_subsets)
    m = len(ix_subsets[class_nothing_to_add]) / target_ratios[class_nothing_to_add]
    total_to_add = m - n
    print(n, math.floor(m))

    examples_to_add = {k: math.floor(target_ratios[k] * m - len(ix_subsets[k])) for k in t_keys}
    print(examples_to_add)  # so we need to add more neutral examples and more positiev ones

    # downsampling would be set 0 the maximum amount of negative ones

    # now select all the indices, with replacement because it can be more than double
    new_samples = []
    for k in t_keys:
        new_samples.extend(random.Random(args.seed).choices(ix_subsets[k], k=examples_to_add[k]))
    print(len(new_samples))

    # now add all new samples to the dataset and shuffle it

    new_sentence_pairs = copy(sentence_pairs)
    new_labels = labels.copy()

    for ix in new_samples:
        new_sentence_pairs.append(copy(sentence_pairs[ix]))
        new_labels.append(labels[ix])

    random.Random(args.seed).shuffle(new_sentence_pairs)
    random.Random(args.seed).shuffle(new_labels)

    print(len(set(new_sentence_pairs)))
    print(len(set(sentence_pairs)))

    return new_sentence_pairs, new_labels


def export_dataset_to_xml(fn, sentence_pairs, labels):
    # export in format semeval 2014, incomplete though! just for loading with existing dataloaders for ATSC
    sentences_el = ET.Element('sentences')
    sentimap_reverse = {
        'POS': 'positive',
        'NEU': 'neutral',
        'NEG': 'negative',
        'CONF': 'conflict'
    }

    for ix, (sentence, aspectterm) in enumerate(sentence_pairs):
        # print(sentence)
        sentiment = labels[ix]
        sentence_el = ET.SubElement(sentences_el, 'sentence')
        sentence_el.set('id', str(ix))
        text = ET.SubElement(sentence_el, 'text')
        text.text = str(sentence).strip()
        aspect_terms_el = ET.SubElement(sentence_el, 'aspectTerms')

        aspect_term_el = ET.SubElement(aspect_terms_el, 'aspectTerm')
        aspect_term_el.set('term', aspectterm)
        aspect_term_el.set('polarity', sentimap_reverse[sentiment])
        aspect_term_el.set('from', str('0'))
        aspect_term_el.set('to', str('0'))

    def indent(elem, level=0):
        i = "\n" + level * "  "
        j = "\n" + (level - 1) * "  "
        if len(elem):
            if not elem.text or not elem.text.strip():
                elem.text = i + "  "
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
            for subelem in elem:
                indent(subelem, level + 1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = j
        else:
            if level and (not elem.tail or not elem.tail.strip()):
                elem.tail = j
        return elem

    indent(sentences_el)
    # mydata = ET.dump(sentences_el)
    mydata = ET.tostring(sentences_el)
    with open(fn, "wb") as f:
        # f.write('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>')
        f.write(mydata)
        f.close()


def save_dataset_to_tsv(fn, data):
    pass


sentence_pairs_train_mixed = []
sentence_pairs_trainsplit_mixed = []
sentence_pairs_dev_mixed = []
sentence_pairs_test_mixed = []

labels_train_mixed = []
labels_trainsplit_mixed = []
labels_dev_mixed = []
labels_test_mixed = []

for fn in args.files:

    print(args.output_dir)
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    print(fn)
    sents_train, ats_train, idx2labels = semeval2014term_to_aspectsentiment_hr(fn,
                                                                               remove_conflicting=args.noconfl)

    sentence_pairs_train, labels_train, counts_train = create_sentence_pairs(sents_train, ats_train)

    if args.istrain:
        sents_dev, sents_trainsplit = split_shuffle_array(.1, sents_train, 41)
        ats_dev, ats_trainsplit = split_shuffle_array(.1, ats_train, 41)

        sentence_pairs_dev, labels_dev, counts_dev = create_sentence_pairs(sents_dev, ats_dev)
        sentence_pairs_trainsplit, labels_trainsplit, counts_trainsplit = create_sentence_pairs(sents_trainsplit,
                                                                                                ats_trainsplit)
        print_dataset_stats('Train', sents_train, sentence_pairs_train, counts_train)
        print_dataset_stats('Dev', sents_dev, sentence_pairs_dev, counts_dev)
        print_dataset_stats('TrainSplit', sents_trainsplit, sentence_pairs_trainsplit, counts_trainsplit)

        sentence_pairs_trainsplit_mixed += sentence_pairs_trainsplit
        sentence_pairs_train_mixed += sentence_pairs_train
        sentence_pairs_dev_mixed += sentence_pairs_dev

        labels_trainsplit_mixed += labels_trainsplit
        labels_train_mixed += labels_train
        labels_dev_mixed += labels_dev

        if len(args.files) == 1:
            if args.upsample:
                distro_arr = args.upsample.split(' ')
                pos = float(distro_arr[0])
                neg = float(distro_arr[1])
                neu = float(distro_arr[2])
                assert pos + neg + neu == 1.0, 'upsampling target distribution does not sum to 1'

                target_distro = {'POS': pos, 'NEG': neg, 'NEU': neu}
                print('Target Sampling Distribution for Training Set:', target_distro)
                sentence_pairs_train, labels_train = upsample_data(sentence_pairs_train, labels_train, target_ratios=target_distro)

            export_dataset_to_xml(args.output_dir + '/train.xml', sentence_pairs_train, labels_train)
            export_dataset_to_xml(args.output_dir + '/dev.xml', sentence_pairs_dev, labels_dev)
            export_dataset_to_xml(args.output_dir + '/train_split.xml', sentence_pairs_trainsplit, labels_trainsplit)

    else:

        sentence_pairs_test_mixed += sentence_pairs_train
        labels_test_mixed += labels_train

        print_dataset_stats('Test', sents_train, sentence_pairs_train, counts_train)
        if len(args.files) == 1:
            export_dataset_to_xml(args.output_dir + '/test.xml', sentence_pairs_train, labels_train)

if len(args.files) > 1:

    if args.istrain:
        export_dataset_to_xml(args.output_dir + '/train.xml', sentence_pairs_train_mixed, labels_train_mixed)
        export_dataset_to_xml(args.output_dir + '/dev.xml', sentence_pairs_dev_mixed, labels_dev_mixed)
        export_dataset_to_xml(args.output_dir + '/train_split.xml', sentence_pairs_trainsplit_mixed,
                              labels_trainsplit_mixed)
    else:
        export_dataset_to_xml(args.output_dir + '/test.xml', sentence_pairs_test_mixed, labels_test_mixed)

After running the code above I have this result:

enter image description here

For English text it works just fine. Could someone help me to fix this and get normal text?

3
  • 3
    Have you tried ET.tostring(sentences_el, encoding='UTF-8')? Note: your output is valid XML with correctly encoded text, it's just not human-readable. XML isn't quite meant to be human-readable.
    – lenz
    Dec 18, 2019 at 8:00
  • Yes, it helped, thank you very much! Dec 19, 2019 at 1:10
  • Please post the example as text, not an image. You'll get more answers if we can cut-n-paste the sample input. Dec 20, 2019 at 6:34

1 Answer 1

1
ET.tostring(sentences_el, encoding='UTF-8')
1
  • 1
    Please dont post code only answer, try to add some context of how this will solve the problem in question. Dec 18, 2019 at 17:35

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.