empathy/dataset_formatting.py at main · ddukoski/empathy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import csv

import pandas as pd
from preprocessing.preprocessing_frames import *


def preprocess_dataset(ds):
    if type(ds) is pd.DataFrame:
        data = ds
    else:
        data = pd.read_csv(ds)
        data['pixels'] = data['pixels'].apply(lambda px: np.fromstring(px, sep=' '))

    data.columns = ['emotion', 'pixels']

    data['pixels'] = data['pixels'].apply(lambda image: preprocess(image, False))
    data['pixels'] = data['pixels'].apply(lambda arr: ' '.join(map(str, arr.flatten())))
    return data


def read_data(arb_csv_file):
    """
    :param arb_csv_file: A datasets in comma-seperated-values format
    :return: A 2-D numpy array containing the datasets, each picture classified per element of the array
    """
    with open(arb_csv_file) as filename:
        em_iterator = csv.reader(filename)
        return list(em_iterator)


if __name__ == '__main__':
    full_ds = read_data('datasets/fer2013.csv')[1:]

    train = list()
    public_test = list()
    private_test = list()

    for sample in full_ds:
        emotion = sample[0]
        pixels = sample[1]
        type_use = sample[-1]

        if type_use == 'Training':
            train.append([emotion, pixels])
        elif type_use == 'PrivateTest':
            private_test.append([emotion, pixels])
        else:
            public_test.append([emotion, pixels])

    training_dataframe = pd.DataFrame(train, columns=['emotion', 'pixels'])
    priv_test_dataframe = pd.DataFrame(private_test, columns=['emotion', 'pixels'])
    pub_test_dataframe = pd.DataFrame(public_test, columns=['emotion', 'pixels'])

    training_dataframe.to_csv('datasets/train.csv', index=False)
    priv_test_dataframe.to_csv('datasets/private_test.csv', index=False)
    pub_test_dataframe.to_csv('datasets/public_test.csv', index=False)

    # preprocessed_train_data = preprocess_dataset('datasets/train.csv')
    # preprocessed_train_data.to_csv('datasets/train.csv', index=False)
    #
    preprocessed_private_data = preprocess_dataset('datasets/private_test.csv')
    preprocessed_private_data.to_csv('datasets/private_test.csv', index=False)
    #
    # preprocessed_validation_data = preprocess_dataset('datasets/public_test.csv')
    # preprocessed_validation_data.to_csv('datasets/public_test.csv', index=False)