input_output.read_config

  1"""Parse config files."""
  2from misc import read_input_csv as ricsv
  3from copy import deepcopy
  4from input_output.organize import Organize_input
  5import tomli
  6import tomli_w
  7import yaml
  8from yaml.loader import FullLoader
  9import numpy as np
 10
 11
 12def convert_txt_to_yaml(init_file):
 13    # Read .pipt or .popt file
 14    pr, fwd = read_txt(init_file)
 15
 16    # Write dictionaries to yaml file with same base file name
 17    new_file = change_file_extension(init_file, 'yaml')
 18    with open(new_file, 'wb') as f:
 19        if 'daalg' in pr:
 20            yaml.dump({'dataassim': pr, 'fwdsim': fwd}, f)
 21        else:
 22            yaml.dump({'optim': pr, 'fwdsim': fwd}, f)
 23
 24
 25def read_yaml(init_file):
 26    """
 27    Read .yaml input file, parse and return dictionaries for PIPT/POPT.
 28
 29    Parameters
 30    ----------
 31    init_file : str
 32        .yaml file
 33
 34    Returns
 35    -------
 36    keys_da : dict
 37        Parsed keywords from dataassim
 38    keys_fwd : dict
 39        Parsed keywords from fwdsim
 40    """
 41    # Make a !ndarray tag to convert a sequence to np.array
 42    def ndarray_constructor(loader, node):
 43        array = loader.construct_sequence(node)
 44        return np.array(array)
 45
 46    # Add constructor to yaml with tag !ndarray
 47    yaml.add_constructor('!ndarray', ndarray_constructor)
 48
 49    # Read
 50    with open(init_file, 'rb') as fid:
 51        y = yaml.load(fid, Loader=FullLoader)
 52
 53    # Check for dataassim and fwdsim
 54    if 'optim' in y.keys():
 55        keys_pr = y['optim']
 56        check_mand_keywords_opt(keys_pr)
 57    elif 'dataassim' in y.keys():
 58        keys_pr = y['datasssim']
 59        check_mand_keywords_da(keys_pr)
 60    else:
 61        raise KeyError
 62    if 'fwdsim' in y.keys():
 63        keys_fwd = y['fwdsim']
 64    else:
 65        raise KeyError
 66
 67    # Organize keywords
 68    org = Organize_input(keys_pr, keys_fwd)
 69    org.organize()
 70
 71    return org.get_keys_pr(), org.get_keys_fwd()
 72
 73
 74def convert_txt_to_toml(init_file):
 75    # Read .pipt or .popt file
 76    pr, fwd = read_txt(init_file)
 77
 78    # Write dictionaries to toml file with same base file name
 79    new_file = change_file_extension(init_file, 'toml')
 80    with open(new_file, 'wb') as f:
 81        if 'daalg' in pr:
 82            tomli_w.dump({'dataassim': pr, 'fwdsim': fwd}, f)
 83        else:
 84            tomli_w.dump({'optim': pr, 'fwdsim': fwd}, f)
 85
 86
 87def read_toml(init_file):
 88    """
 89    Read .toml configuration file, parse and output dictionaries for PIPT/POPT
 90
 91    Parameters
 92    ----------
 93    init_file : str
 94        toml configuration file
 95    """
 96    # Read
 97    with open(init_file, 'rb') as fid:
 98        t = tomli.load(fid)
 99
100    # Check for dataassim and fwdsim
101    if 'ensemble' in t.keys():
102        keys_en = t['ensemble']
103        check_mand_keywords_en(keys_en)
104    else:
105        keys_en = None
106    if 'optim' in t.keys():
107        keys_pr = t['optim']
108        check_mand_keywords_opt(keys_pr)
109    elif 'dataassim' in t.keys():
110        keys_pr = t['dataassim']
111        check_mand_keywords_da(keys_pr)
112    else:
113        raise KeyError
114    if 'fwdsim' in t.keys():
115        keys_fwd = t['fwdsim']
116    else:
117        raise KeyError
118
119    # Organize keywords
120    org = Organize_input(keys_pr, keys_fwd, keys_en)
121    org.organize()
122
123    return org.get_keys_pr(), org.get_keys_fwd(), org.get_keys_en()
124
125
126def read_txt(init_file):
127    """
128    Read a PIPT or POPT input file (.pipt or .popt), parse and output dictionaries for data assimilation or
129    optimization,  and simulator classes.
130
131    Parameters
132    ----------
133    init_file: str
134        PIPT init. file containing info. to run the inversion algorithm
135
136    Returns
137    -------
138    keys_pr : dict
139        Parsed keywords from DATAASSIM or OPTIM
140    keys_fwd : dict
141        Parsed keywords from FWDSSIM
142    """
143
144    # Check for .pipt suffix
145    if not init_file.endswith('.pipt') and not init_file.endswith('.popt'):
146        raise FileNotFoundError(f'No PIPT or POPT input file (.pipt or .popt) found! If {init_file} is  '
147                                f'a PIPT or POPT input file, change suffix to .pipt or .popt')
148
149    # Read the init file and output lines without comments (lines starting with '#')
150    lines = read_clean_file(init_file)
151
152    # Find where the separate parts are located in the file. FWDSIM will always be a part, but the
153    # inversion/optimiztation part may be DATAASSIM or OPTIM
154    prind = None
155    pr_part = None
156    fwdsimind = None
157    for i in range(len(lines)):
158        if lines[i].strip().lower() == 'dataassim' or lines[i].strip().lower() == 'optim':
159            prind = i
160            pr_part = lines[i].strip().lower()
161        elif lines[i].strip().lower() == 'fwdsim':
162            fwdsimind = i
163
164    # Split the file into the two separate parts. Each part will (only) contain the keywords of each part:
165    if prind < fwdsimind:  # Data assim. part is the first part of file
166        lines_pr = lines[2:fwdsimind]
167        lines_fwd = lines[fwdsimind + 2:]
168    else:  # Fwd sim. part is the first part of file
169        lines_fwd = lines[2:prind]
170        lines_pr = lines[prind + 2:]
171
172    # Get rid of empty lines in lines_pr and lines_fwd
173    clean_lines_pr = remove_empty_lines(lines_pr)
174    clean_lines_fwd = remove_empty_lines(lines_fwd)
175
176    # Assign the keys and values to different dictionaries depending on whether we have data assimilation (DATAASSIM)
177    # or optimization (OPTIM). FWDSIM info is always assigned to keys_fwd
178    keys_pr = None
179    if pr_part == 'dataassim':
180        keys_pr = parse_keywords(clean_lines_pr)
181        check_mand_keywords_da(keys_pr)
182    elif pr_part == 'optim':
183        keys_pr = parse_keywords(clean_lines_pr)
184        check_mand_keywords_opt(keys_pr)
185    keys_fwd = parse_keywords(clean_lines_fwd)
186    check_mand_keywords_fwd(keys_fwd)
187
188    org = Organize_input(keys_pr, keys_fwd)
189    org.organize()
190
191    return org.get_keys_pr(), org.get_keys_fwd()
192
193
194def read_clean_file(init_file):
195    """
196    Read PIPT init. file and lines that are not comments (marked with octothorpe)
197
198    Parameters
199    ----------
200    init_file: str
201        Name of file to remove all comments. WHOLE filename needed (with suffix!)
202
203    Returns
204    -------
205    lines: list
206        Lines from init. file converted to list entries
207    """
208    # Read file except lines starting with an octothorpe (#) and return the python variable
209    with open(init_file, 'r') as f:
210        lines = [line for line in f.readlines() if not line.startswith('#')]
211
212    # Return clean lines
213    return lines
214
215
216def remove_empty_lines(lines):
217    """
218    Small method for finding empty lines in a read file.
219
220    Parameters
221    ----------
222    lines: list
223        List of lines from a file
224
225    Returns
226    -------
227    lines_clean: list
228        List of clean lines (without empty entries)
229    """
230    # Loop over lines to find '\n'
231    sep = []
232    for i in range(len(lines)):
233        if lines[i] == '\n':
234            sep.append(i)
235
236    # Make clean output
237    lines_clean = []
238    for i in range(len(sep)):
239        if i == 0:
240            lines_clean.append(lines[0:sep[i]])
241        else:
242            lines_clean.append(lines[sep[i-1] + 1:sep[i]])
243
244    # Return
245    return lines_clean
246
247
248def parse_keywords(lines):
249    """
250    Here we parse the lines in the init. file to a Python dictionary. The keys of the dictionary is the keywords
251    in the PIPT init. file, and the information in each keyword is stored in each key of the
252    dictionary. To know how the keyword-information is organized in the keys of the dictionary, confront the
253    manual located in the doc folder.
254
255    Parameters
256    ----------
257    lines: list
258        List of (clean) lines from the PIPT init. file.
259
260    Returns
261    -------
262    keys: dict
263        Dictionary with all info. from the init. file.
264    """
265    # Init. the dictionary
266    keys = {}
267
268    # Loop over all input keywords and store in the dictionary.
269    for i in range(len(lines)):
270        if lines[i] != []:  # Check for empty list (corresponds to empty line in file)
271            try:  # Try first to store the info. in keyword as float in a 1D list
272                # A scalar, which we store as scalar...
273                if len(lines[i][1:]) == 1 and len(lines[i][1:][0].split()) == 1:
274                    keys[lines[i][0].strip().lower()] = float(lines[i][1:][0])
275                else:
276                    keys[lines[i][0].strip().lower()] = [float(x) for x in lines[i][1:]]
277            except:
278                try:  # Store as float in 2D list
279                    if len(lines[i][1:]) == 1:  # Check if it is actually a 1D array disguised as 2D
280                        keys[lines[i][0].strip().lower()] = \
281                            [float(x) for x in lines[i][1:][0].split()]
282                    else:  # if not store as 2D list
283                        keys[lines[i][0].strip().lower()] = \
284                            [[float(x) for x in col.split()] for col in lines[i][1:]]
285                except:  # Keyword contains string(s), not floats
286                    if len(lines[i][1:]) == 1:  # If 1D list
287                        # If it is a scalar store as single input
288                        if len(lines[i][1:][0].split('\t')) == 1:
289                            keys[lines[i][0].strip().lower()] = lines[i][1:][0].strip().lower()
290                        else:  # Store as 1D list
291                            keys[lines[i][0].strip().lower()] = \
292                                [x.rstrip('\n').lower()
293                                 for x in lines[i][1:][0].split('\t') if x != '']
294                    else:  # It is a 2D list
295                        # Check each row in 2D list. If it is single column (i.e., one string per row),
296                        # we make it a 1D list of strings; if not, we make it a 2D list of strings.
297                        one_col = True
298                        for j in range(len(lines[i][1:])):
299                            if len(lines[i][1:][j].split('\t')) > 1:
300                                one_col = False
301                                break
302                        if one_col is True:  # Only one column
303                            keys[lines[i][0].strip().lower()] = \
304                                [x.rstrip('\n').lower() for x in lines[i][1:]]
305                        else:  # Store as 2D list
306                            keys[lines[i][0].strip().lower()] = \
307                                [[x.rstrip('\n').lower() for x in col.split('\t') if x != '']
308                                    for col in lines[i][1:]]
309
310    # Need to check if there are any only-string-keywords that actually contains floats, and convert those to
311    # floats (the above loop only handles pure float or pure string input, hence we do a quick fix for mixed
312    # lists here)
313    # Loop over all keys in dict. and check every "pure" string keys for floats
314    for i in keys:
315        if isinstance(keys[i], list):  # Check if key is a list
316            if isinstance(keys[i][0], list):  # Check if it is a 2D list
317                for j in range(len(keys[i])):  # Loop over all sublists
318                    # Check sublist for strings
319                    if all(isinstance(x, str) for x in keys[i][j]):
320                        for k in range(len(keys[i][j])):  # Loop over enteries in sublist
321                            try:  # Try to make float
322                                keys[i][j][k] = float(keys[i][j][k])  # Scalar
323                            except:
324                                try:  # 1D array
325                                    keys[i][j][k] = [float(x)
326                                                     for x in keys[i][j][k].split()]
327                                except:  # If it is actually a string, pass over
328                                    pass
329            else:  # It is a 1D list
330                # Check if list only contains strings
331                if all(isinstance(x, str) for x in keys[i]):
332                    for j in range(len(keys[i])):  # Loop over all entries in list
333                        try:  # Try to make float
334                            keys[i][j] = float(keys[i][j])
335                        except:
336                            try:
337                                keys[i][j] = [float(x) for x in keys[i][j].split()]
338                            except:  # If it is actually a string, pass over
339                                pass
340
341    # Return dict.
342    return keys
343
344
345def check_mand_keywords_fwd(keys_fwd):
346    """Check for mandatory keywords in `FWDSIM` part, and output error if they are not present"""
347
348    # Mandatory keywords in FWDSIM
349    assert 'parallel' in keys_fwd, 'PARALLEL not in FWDSIM!'
350    assert 'datatype' in keys_fwd, 'DATATYPE not in FWDSIM!'
351
352
353def check_mand_keywords_da(keys_da):
354    """Check for mandatory keywords in `DATAASSIM` part, and output error if they are not present"""
355
356    # Mandatory keywords in DATAASSIM
357    assert 'truedataindex' in keys_da, 'TRUEDATAINDEX not in DATAASSIM!'
358    assert 'assimindex' in keys_da, 'ASSIMINDEX not in DATAASSIM!'
359    assert 'truedata' in keys_da, 'TRUEDATA not in DATAASSIM!'
360    assert 'datavar' in keys_da, 'DATAVAR not in DATAASSIM!'
361    assert 'obsname' in keys_da, 'OBSNAME not in DATAASSIM!'
362    assert 'energy' in keys_da, 'ENERGY not in DATAASSIM!'
363
364
365def check_mand_keywords_opt(keys_opt):
366    """Check for mandatory keywords in `OPTIM` part, and output error if they are not present"""
367pass
368
369
370def check_mand_keywords_en(keys_en):
371    """Check for mandatory keywords in `ENSEMBLE` part, and output error if they are not present"""
372
373    # Mandatory keywords in ENSEMBLE
374    assert 'ne' in keys_en, 'NE not in ENSEMBLE!'
375    assert 'state' in keys_en, 'STATE not in ENSEMBLE!'
376    if 'importstaticvar' not in keys_en:
377        assert filter(list(keys_en.keys()),
378                      'prior_*') != [], 'No PRIOR_<STATICVAR> in DATAASSIM'
379
380def change_file_extension(filename, new_extension):
381    if '.' in filename:
382        name, old_extension = filename.rsplit('.', 1)
383        new_filename = name + '.' + new_extension
384    else:
385        new_filename = filename + '.' + new_extension
386    return new_filename