misc.read_input_csv

  1"""
  2File for reading CSV files and returning a 2D list
  3"""
  4import pandas as pd
  5import numpy as np
  6
  7
  8def read_data_csv(filename, datatype, truedataindex):
  9    """
 10    Parameters
 11    ----------
 12    filename:
 13        Name of csv-file
 14    datatype:
 15        List of data types as strings
 16    truedataindex:
 17        List of where the "TRUEDATA" has been extracted (e.g., at which time, etc)
 18
 19    Returns
 20    -------
 21    some-type:
 22        List of observed data
 23    """
 24
 25    df = pd.read_csv(filename)  # Read the file
 26
 27    imported_data = []  # Initialize the 2D list of csv data
 28    tlength = len(truedataindex)
 29    dnumber = len(datatype)
 30
 31    if df.columns[0] == 'header_both':  # csv file has column and row headers
 32        pos = [None] * dnumber
 33        for col in range(dnumber):
 34            # find index of data type in csv file header
 35            pos[col] = df.columns.get_loc(datatype[col])
 36        for t in truedataindex:
 37            row = df[df['header_both'] == t]  # pick row corresponding to truedataindex
 38            row = row.values[0]  # select the values of the dataframe row
 39            csv_data = [None] * dnumber
 40            for col in range(dnumber):
 41                if (not type(row[pos[col]]) == str) and (np.isnan(row[pos[col]])):  # do not check strings
 42                    csv_data[col] = 'n/a'
 43                else:
 44                    try:  # Making a float
 45                        csv_data[col] = float(row[pos[col]])
 46                    except:  # It is a string
 47                        csv_data[col] = row[pos[col]]
 48            imported_data.append(csv_data)
 49    else:  # No row headers (the rows in the csv file must correspond to the order in truedataindex)
 50        if tlength == df.shape[0]:  # File has column headers
 51            pos = [None] * dnumber
 52            for col in range(dnumber):
 53                # Find index of the header in datatype
 54                pos[col] = df.columns.get_loc(datatype[col])
 55        # File has no column headers (columns must correspond to the order in datatype)
 56        elif tlength == df.shape[0]+1:
 57            # First row has been misinterpreted as header, so we read first row again:
 58            temp = pd.read_csv(filename, header=None, nrows=1).values[0]
 59            pos = list(range(df.shape[1]))  # Assume the data is in the correct order
 60            csv_data = [None] * len(temp)
 61            for col in range(len(temp)):
 62                if (not type(temp[col]) == str) and (np.isnan(temp[col])):  # do not check strings
 63                    csv_data[col] = 'n/a'
 64                else:
 65                    try:  # Making a float
 66                        csv_data[col] = float(temp[col])
 67                    except:  # It is a string
 68                        csv_data[col] = temp[col]
 69            imported_data.append(csv_data)
 70
 71        for rows in df.values:
 72            csv_data = [None] * dnumber
 73            for col in range(dnumber):
 74                if (not type(rows[pos[col]]) == str) and (np.isnan(rows[pos[col]])):  # do not check strings
 75                    csv_data[col] = 'n/a'
 76                else:
 77                    try:  # Making a float
 78                        csv_data[col] = float(rows[pos[col]])
 79                    except:  # It is a string
 80                        csv_data[col] = rows[pos[col]]
 81            imported_data.append(csv_data)
 82
 83    return imported_data
 84
 85
 86def read_var_csv(filename, datatype, truedataindex):
 87    """
 88    Parameters
 89    ----------
 90    filename : str
 91        Name of the CSV file.
 92
 93    datatype : list
 94        List of data types as strings.
 95
 96    truedataindex : list
 97        List of indices where the "TRUEDATA" has been extracted.
 98
 99    Returns
100    -------
101    imported_var : list
102        List of variances.
103    """
104
105    df = pd.read_csv(filename)  # Read the file
106
107    imported_var = []  # Initialize the 2D list of csv data
108    tlength = len(truedataindex)
109    dnumber = len(datatype)
110
111    if df.columns[0] == 'header_both':  # csv file has column and row headers
112        pos = [None] * dnumber
113        for col in range(dnumber):
114            # find index of data type in csv file header
115            pos[col] = df.columns.get_loc(datatype[col])
116        for t in truedataindex:
117            row = df[df['header_both'] == t]  # pick row
118            row = row.values[0]  # select the values of the dataframe
119            csv_data = [None] * 2 * dnumber
120            for col in range(dnumber):
121                csv_data[2*col] = row[pos[col]]
122                try:  # Making a float
123                    csv_data[2*col+1] = float(row[pos[col]]+1)
124                except:  # It is a string
125                    csv_data[2*col+1] = row[pos[col]+1]
126            # Make sure the string input is lowercase
127            csv_data[0::2] = [x.lower() for x in csv_data[0::2]]
128            imported_var.append(csv_data)
129    else:  # No row headers (the rows in the csv file must correspond to the order in truedataindex)
130        if tlength == df.shape[0]:  # File has column headers
131            pos = [None] * dnumber
132            for col in range(dnumber):
133                # Find index of datatype in csv file header
134                pos[col] = df.columns.get_loc(datatype[col])
135        # File has no column headers (columns must correspond to the order in datatype)
136        elif tlength == df.shape[0]+1:
137            # First row has been misinterpreted as header, so we read first row again:
138            temp = pd.read_csv(filename, header=None, nrows=1).values[0]
139            # Make sure the string input is lowercase
140            temp[0::2] = [x.lower() for x in temp[0::2]]
141            # Assume the data is in the correct order
142            pos = list(range(0, df.shape[1], 2))
143            csv_data = [None] * len(temp)
144            for col in range(dnumber):
145                csv_data[2 * col] = temp[2 * col]
146                try:  # Making a float
147                    csv_data[2*col+1] = float(temp[2*col+1])
148                except:  # It is a string
149                    csv_data[2*col+1] = temp[2*col+1]
150            imported_var.append(csv_data)
151
152        for rows in df.values:
153            csv_data = [None] * 2 * dnumber
154            for col in range(dnumber):
155                csv_data[2*col] = rows[2*col]
156                try:  # Making a float
157                    csv_data[2*col+1] = float(rows[pos[col]+1])
158                except:  # It is a string
159                    csv_data[2*col+1] = rows[pos[col]+1]
160            # Make sure the string input is lowercase
161            csv_data[0::2] = [x.lower() for x in csv_data[0::2]]
162            imported_var.append(csv_data)
163
164    return imported_var