#!/usr/bin/env python

"""This script loads model data."""

# ############################ LICENSE INFORMATION ############################
# This file is part of the E3 RESOLVE Model.

# Copyright (C) 2019 Energy and Environmental Economics, Inc.
# For contact information, go to www.ethree.com

# The E3 RESOLVE Model is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# The E3 RESOLVE Model is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with the E3 RESOLVE Model (in the file LICENSE.TXT). If not,
# see <http://www.gnu.org/licenses/>.
# #############################################################################

import ast
import logging
import os
import pandas as pd
import pyomo.environ as pyo


def convert_index_to_timepoint(timepoint_mapping, indices):
    """Convert list of tuple indices to list of timepoints.

    Args:
        timepoint_mapping (pd.DataFrame): Dataframe indexed by timepoint with period, month, day, and hour_of_day values
        indices (List[tuple]): List of indices (period, day, hour_of_day_from, hour_of_day_to)

    Returns:
        list: List of timepoint IDs
    """
    # find applicable timepoints
    timepoints = [
        timepoint_mapping.loc[
            (timepoint_mapping['PERIODS'] == int(period)) &
            (timepoint_mapping['DAYS'] == int(day)) &
            (timepoint_mapping['HOURS_OF_DAY'] == int(hour_of_day)), :
        ].index[0]
        for (period, day, hour_of_day_from, hour_of_day_to) in indices
        for hour_of_day in range(int(hour_of_day_from), int(hour_of_day_to) + 1)
    ]

    return timepoints


def create_timepoint_mapping(data, namespace=None):
    """Creates a map of timepoints to the sets for PERIODS, MONTHS, DAYS, and HOURS_OF_DAY.

    Args:
        data (pyo.DataPortal): Pyomo DataPortal object (we manually fill its '_data' attribute)
        namespace (str): Optional Pyomo model namespace (defaults to None)

    Returns:
        timepoint_mapping (pd.DataFrame): Dataframe indexed by timepoint with period, month, day, and hour_of_day values
        sets (dict): Dict of unique sets for PERIODS, MONTHS, DAYS, and HOURS_OF_DAY
    """

    # Get mappings between timepoints and period, month, day, and hour
    timepoint_map = {}
    timepoint_map['PERIODS'] = data._data[namespace]['period']
    timepoint_map['MONTHS'] = data._data[namespace]['month']
    timepoint_map['DAYS'] = data._data[namespace]['day']
    timepoint_map['HOURS_OF_DAY'] = data._data[namespace]['hour_of_day']

    # Make a DataFrame to help find the timepoint that matches period, day, hour
    timepoint_mapping = pd.DataFrame(timepoint_map)
    # timepoint_mapping.columns = ['period', 'month', 'day', 'hour_of_day']
    timepoint_mapping.index.name = 'timepoint'

    # Get unique sets
    sets = {}
    sets['PERIODS'] = set(sorted(timepoint_map['PERIODS'].values()))
    sets['MONTHS'] = set(sorted(timepoint_map['MONTHS'].values()))
    sets['DAYS'] = set(sorted(timepoint_map['DAYS'].values()))
    sets['HOURS'] = set(sorted(timepoint_map['HOURS_OF_DAY'].values()))

    return timepoint_mapping, sets


def determine_flexible_param_type(index):

    param, model_object, period, day, hour_of_day_from, hour_of_day_to = index

    # 1. Unindexed Params
    if all(idx == 'None' for idx in [model_object, period, day, hour_of_day_from, hour_of_day_to]):
        return 'Unindexed'

    # 2. Object-indexed Params (indexed only by whatever is in the 'model_object' column)
    elif all(idx == 'None' for idx in [period, day, hour_of_day_from, hour_of_day_to]):
        return 'ObjectIndexed'

    # 3. Object- and period-indexed Params
    elif (
        period != 'None' and
        all(idx == 'None' for idx in [day, hour_of_day_from, hour_of_day_to])
    ):
        return 'ObjectPeriodIndexed'

    # 4. Object- and day-indexed Params
    elif (
        day != 'None' and
        all(idx == 'None' for idx in [period, hour_of_day_from, hour_of_day_to])
    ):
        return 'ObjectDayIndexed'

    # 5. Object- and day-indexed Params
    elif (
        day != 'None' and
        period != 'None' and
        all(idx == 'None' for idx in [hour_of_day_from, hour_of_day_to])
    ):
        return 'ObjectPeriodDayIndexed'

    # 6. Object- and timepoint-indexed Params
    else:
        return 'ObjectTimepointIndexed'


def parse_flexible_params(data, inputs_directory, resolve_model, namespace=None):
    """Reads flexible_params file and adds them directly to the DataPortal's internal dictionary of values.

    Warning: Reading timepoint-indexed Params is significantly slower than others,
    as we do more processing in the script to figure out the timepoint IDs to apply to.

    This method can currently accommodate five types of Params:
        1. Unindexed Params
        2. Object-indexed Params (indexed only by whatever is in the 'model_object' column)
        3. Object- and period-indexed Params
        4. Object- and day-indexed Params
        5. Object- and timepoint-indexed Params

    Args:
        data (pyo.DataPortal): Pyomo DataPortal object (we manually fill its '_data' attribute)
        inputs_directory (str): Name of case inputs directory
        resolve_model (ResolveModel): AbstractModel formulation for RESOLVE to check data types
        namespace (str): Optional Pyomo model namespace (defaults to None)

    Raises:
        AttributeError: If parameter data already exists in DataPortal._data attribute

    Returns:
        data (pyo.DataPortal): Pyomo DataPortal object (after additional parameter values added)
        flexible_params (pd.DataFrame)
    """
    # Create custom logger for warnings
    logger = logging.getLogger('resolve_model.load_data')

    # get a list of "acceptable" flexible params, which should be documented as-such in the formulation
    acceptable_flexible_params = sorted([
        name
        for name, attr in resolve_model.__dict__.items()
        if isinstance(attr, pyo.Param) and '#flexibleparam' in str(attr.doc)
    ])

    # Get timepoints mapping
    timepoint_mapping, sets = create_timepoint_mapping(data)

    # Read the CSV to start parsing
    flexible_params = pd.read_csv(
        os.path.join(inputs_directory, "flexible_params.csv"),
        header=0,
        index_col=list(range(0, 6))
    )

    params_to_create = flexible_params.index.get_level_values(level=0).unique()

    # Do some data validation
    for param in params_to_create:
        # Raise an error if CSV included components not known as flexible params
        if param not in acceptable_flexible_params:
            raise AttributeError(
                'Parameter \'{param}\' is not a known flexible parameter. \nAccceptable flexible param names are: \n - {names}'
                .format(param=param, names='\n - '.join(acceptable_flexible_params))
            )

        # Raise an error if any flexible params already exist in the DataPortal object
        if param in data._data[namespace].keys():
            raise AttributeError('Data for parameter {} is already loaded via data.load()'.format(param))
        else:
            # Initialize the nested dictionary with flexible params
            data._data[namespace][param] = {}

    # parse flexible params into DataPortal's _data dictionary
    for index, value in flexible_params.iterrows():

        # split tuple index for clarity
        param, model_object, period, day, hour_of_day_from, hour_of_day_to = index

        # use ResolveModel Param domain to coerce type before putting in data dictionary
        if getattr(resolve_model, param).domain.name in ['Boolean', 'Binary']:
            if value.dtypes == "float64":
                value = value.astype(int).astype('bool')
            else:
                value = value.map(ast.literal_eval).astype('bool')
        elif 'Integers' in getattr(resolve_model, param).domain.name:
            value = value.astype('int')
        else:
            try:
                value = value.astype('float')
            except ValueError:
                pass

        # get only the value inside the pd.Series
        value = value.values[0]

        # skip the flexible param if it's just the default value to reduce slowness
        if not getattr(resolve_model, param)._default_val == value:

            # 1. Unindexed Params
            if determine_flexible_param_type(index) == 'Unindexed':
                data._data[namespace][param].update(
                    {None: value}
                )

            # 2. Object-indexed Params (indexed only by whatever is in the 'model_object' column)
            elif determine_flexible_param_type(index) == 'ObjectIndexed':
                data._data[namespace][param].update(
                    {model_object: value}
                )

            # 3. Object- and period-indexed Params
            elif determine_flexible_param_type(index) == 'ObjectPeriodIndexed':
                # Skip if index is not in active indices (i.e., if user put in data for a year not modeled)
                if period != 'All' and int(period) not in sets['PERIODS']:
                    logger.warning(
                        "Skipping value {} for flexible parameter `{}` for non-modeled period {}"
                        .format(value, param, period)
                    )
                    continue

                if period == 'All':
                    indices = sets['PERIODS']
                else:
                    indices = [int(period)]

                data._data[namespace][param].update(
                    {(model_object, idx): value for idx in indices}
                )

            # 4. Object- and day-indexed Params
            elif determine_flexible_param_type(index) == 'ObjectDayIndexed':
                # Skip if index is not in active indices (i.e., if user put in data for a year not modeled)
                if day != 'All' and int(day) not in sets['DAYS']:
                    logger.warning(
                        "Skipping value {} for flexible parameter `{}` for non-modeled day {}"
                        .format(value, param, day)
                    )
                    continue

                if day == 'All':
                    indices = sets['DAYS']
                else:
                    indices = [int(day)]

                data._data[namespace][param].update(
                    {(model_object, idx): value for idx in indices}
                )

            # 5. Object-, period-, and day-indexed Params
            elif determine_flexible_param_type(index) == 'ObjectPeriodDayIndexed':
                if (day == 'All') and (period == 'All'):
                    indices = [
                        (period_in_set, day_in_set)
                        for day_in_set in sets['DAYS']
                        for period_in_set in sets['PERIODS']
                    ]

                elif (day == 'All') and (not period == 'All'):
                    indices = [
                        (int(period), day_in_set)
                        for day_in_set in sets['DAYS']
                    ]
                elif (not day == 'All') and (period == 'All'):
                    indices = [
                        (period_in_set, int(day))
                        for period_in_set in sets['PERIODS']
                    ]
                else:
                    indices = [(int(period), int(day))]

                data._data[namespace][param].update(
                    {(model_object, period, day): value for period, day in indices}
                )

            # 6. Object- and timepoint-indexed Params
            elif determine_flexible_param_type(index) == 'ObjectTimepointIndexed':
                try:
                    assert int(hour_of_day_from) in sets['HOURS'] and int(hour_of_day_to) in sets['HOURS']
                except AssertionError as error:
                    raise ValueError('Hour range set for {index} is invalid.'.format(index=index))

                # Skip if index is not in active indices (i.e., if user put in data for a year not modeled)
                if (
                    (period != 'All' and int(period) not in sets['PERIODS']) or
                    (day != 'All' and int(day) not in sets['DAYS'])
                ):
                    logger.warning(
                        "Skipping value {} for flexible parameter `{}` for non-modeled timepoint {}"
                        .format(value, param, index)
                    )
                    continue

                if (day == 'All') and (period == 'All'):
                    indices = [
                        (period_in_set, day_in_set, hour_of_day_from, hour_of_day_to)
                        for day_in_set in sets['DAYS']
                        for period_in_set in sets['PERIODS']
                    ]

                elif (day == 'All') and (not period == 'All'):
                    indices = [
                        (int(period), day_in_set, hour_of_day_from, hour_of_day_to)
                        for day_in_set in sets['DAYS']
                    ]
                elif (not day == 'All') and (period == 'All'):
                    indices = [
                        (period_in_set, int(day), hour_of_day_from, hour_of_day_to)
                        for period_in_set in sets['PERIODS']
                    ]
                else:
                    indices = [(int(period), int(day), hour_of_day_from, hour_of_day_to)]

                indices = convert_index_to_timepoint(timepoint_mapping, indices)

                data._data[namespace][param].update(
                    {(model_object, idx): value for idx in indices}
                )

            else:
                raise ValueError('Flexible param not recognized: {param} {index}'.format(param, index))

    return data, flexible_params


def scenario_data(inputs_directory, resolve_model):
    """
    Find and load data for the specified scenario.

    :param inputs_directory: the scenario inputs directory
    :param feature_toggles: toggle to turn on and off the functionality
    :return:
    """

    data = pyo.DataPortal()

    # All timepoints modeled with their associated metadata: which period, month, and day the timepoint is in,
    # and which hour of the day it represents.
    data.load(filename=os.path.join(inputs_directory, "timepoints.csv"),
              index=resolve_model.TIMEPOINTS,
              param=(resolve_model.period,
                     resolve_model.month,
                     resolve_model.day,
                     resolve_model.hour_of_day)
              )


    # Flexible param functionality reads in a CSV and adds directly to DataPortal's '_dict' attribute
    data, flexible_params = parse_flexible_params(data, inputs_directory, resolve_model)

    # The weight/discount factor applied to costs occurring in each period,
    # and the number of years represented by each period.
    data.load(filename=os.path.join(inputs_directory, "period_discount_factors.csv"),
              index=resolve_model.PERIODS,
              param=(resolve_model.discount_factor,
                     resolve_model.years_in_period)
              )

    # The weight associated with each day in RESOLVE; should sum up to 365.
    data.load(filename=os.path.join(inputs_directory, "day_weights.csv"),
              index=resolve_model.DAYS,
              param=resolve_model.day_weight
              )

    # The zones modeled, the spinning reserve requirements (if any) for each zone, and various flags.
    data.load(filename=os.path.join(inputs_directory, "zones.csv"),
              index=resolve_model.ZONES,
              param=(resolve_model.spin_reserve_fraction_of_load,
                     resolve_model.include_in_load_following
                     )
              )

    # The input load in each zone in each timepoint.
    # Indexed by ZONES and TIMEPOINTS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "zone_timepoint_params.csv"),
              param=resolve_model.input_load_mw
              )

    # The regulation and load-following reserve requirements in each timepoint.
    data.load(filename=os.path.join(inputs_directory, "reserve_timepoint_requirements.csv"),
              index=resolve_model.TIMEPOINTS,
              param=(resolve_model.upward_reg_req,
                     resolve_model.downward_reg_req,
                     resolve_model.upward_lf_reserve_req,
                     resolve_model.downward_lf_reserve_req,
                     resolve_model.min_gen_committed_mw,
                     resolve_model.freq_resp_total_req_mw,
                     resolve_model.freq_resp_partial_req_mw,
                     resolve_model.spin_reserve_req_mw
                     )
              )

    # All transmission lines with their origin (from) and destination (to) for the positive flow direction,
    # the minimum and maximum flow on the line, a flag for whether the line is ramp-constrained,
    # and a flag for whether a hurdle rate is applied on the line.
    data.load(filename=os.path.join(inputs_directory, "transmission_lines.csv"),
              index=resolve_model.TRANSMISSION_LINES,
              param=(resolve_model.transmission_from,
                     resolve_model.transmission_to,
                     resolve_model.min_flow_planned_mw,
                     resolve_model.max_flow_planned_mw,
                     resolve_model.ramp_constrained,
                     resolve_model.new_build_tx_flag)
              )


    # The contribution of tx lines to each reliability constraint,
    # including existing and new tx contributions with an adder
    if resolve_model.allow_tx_reliability_contribution:
        data.load(filename=os.path.join(inputs_directory, "transmission_reliability_contribution.csv"),
                  index=resolve_model.TRANSMISSION_RELIABILITY_MAP,
                  param=(resolve_model.transmission_direction,
                         resolve_model.existing_tx_reliability_contribution,
                         resolve_model.new_tx_reliability_contribution)
                  )

    # The names of the groups of lines over which simultaneous flow constraints are enforced.
    data.load(filename=os.path.join(inputs_directory, "simultaneous_flow_groups.csv"),
              set=resolve_model.SIMULTANEOUS_FLOW_GROUPS
              )

    # The limits on flow over each simultaneous flow group by period.
    # Indexed by SIMULTANEOUS_FLOW_GROUPS and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "simultaneous_flow_limits.csv"),
              param=resolve_model.simultaneous_flow_limit_mw
              )

    # The line-directions included in each simultaneous flow group.
    data.load(filename=os.path.join(inputs_directory, "simultaneous_flow_group_lines.csv"),
              index=resolve_model.SIMULTANEOUS_FLOW_GROUP_LINES,
              param=resolve_model.direction)

    if resolve_model.transmission_ramp_limit:
        # The up and down ramp limits for each ramp-constrained line for each ramp duration.
        # Indexed by RAMP_CONSTRAINED_TRANSMISSION_LINES and INTERTIE_FLOW_RAMP_DURATIONS in the model formulation
        data.load(filename=os.path.join(inputs_directory, "transmission_ramps.csv"),
                  param=(resolve_model.flow_ramp_up_limit_fraction,
                         resolve_model.flow_ramp_down_limit_fraction)
                  )

    # Hurdle rates (cost per MW of energy flow) on each transmission line by period for both flow directions.
    # Indexed by TRANSMISSION_LINES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "hurdle_rates.csv"),
              param=(resolve_model.positive_direction_hurdle_rate_per_mw,
                     resolve_model.negative_direction_hurdle_rate_per_mw)
              )

    # Defines the set of fuels and the carbon content of each fuel.
    data.load(filename=os.path.join(inputs_directory, "fuels.csv"),
              index=resolve_model.FUELS,
              param=resolve_model.is_blend_fuel
              )

    # The price of each fuel by period and month.
    # Indexed by FUELS, PERIODS, and MONTHS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "fuel_prices.csv"),
              param=resolve_model.fuel_price_per_mmbtu
              )

    # The pairing between base fuels and blend fuels
    # Indexed by BLENDABLE_BASE_FUELS
    if resolve_model.allow_fuel_blending:
        data.load(filename=os.path.join(inputs_directory, "fuel_blending_pairs.csv"),
                  index=resolve_model.BLENDABLE_BASE_FUELS,
                  param=resolve_model.blend_fuel
                  )

    if resolve_model.include_emissions:
        # Types of emissions and their corresponding units (for reporting)
        data.load(filename=os.path.join(inputs_directory,"emission_types.csv"),
                  index=resolve_model.EMISSION_TYPES,
                  param=resolve_model.emission_type_units
                  )

        data.load(filename=os.path.join(inputs_directory, "resource_emissions_groups.csv"),
                  set=resolve_model.RESOURCE_EMISSIONS_GROUPS_MAP
                  )

        # Emissions targets for each period.
        if resolve_model.include_emissions_targets:
            data.load(filename=os.path.join(inputs_directory, "emissions_targets.csv"),
                      index=resolve_model.EMISSIONS_GROUP_CONSTRAINT_PERIODS,
                      param=(
                          resolve_model.emissions_target_units_per_year,
                          resolve_model.emissions_credit_units_per_year)
                      )

        # emissions factors for each resource for each group
        data.load(filename=os.path.join(inputs_directory, "resource_emissions.csv"),
                  param=(
                      resolve_model.resource_emission_factor,
                      resolve_model.resource_emission_factor_blended_fuel,
                      resolve_model.resource_emission_factor_denominator_mmbtu_or_mwh)
                  )

        # The assumed emissions intensity resulting from imports into the ghg target area
        # in each period for each transmission line.
        # Indexed by TRANSMISSION_LINES, EMISSIONS_GROUPS, and PERIODS in the model formulation
        data.load(filename=os.path.join(inputs_directory, "emissions_groups_tx_lines.csv"),
                  param=(resolve_model.emission_factor_positive_direction_per_mwh,
                         resolve_model.emission_factor_negative_direction_per_mwh)
                  )


    # Parameters associated with each thermal resource: the fuel used, and the fuel burn slope and intercept.
    data.load(filename=os.path.join(inputs_directory, "resource_thermal_params.csv"),
              param=(resolve_model.fuel,
                     resolve_model.fuel_burn_slope_mmbtu_per_mwh,
                     resolve_model.fuel_burn_intercept_mmbtu_per_hr)
              )

    # Parameters associated with each unit commitment resource:
    # minimum stable level as fraction of capacity, ramp rate as fraction of capacity,
    # startup and shutdown time (integer hours), unit size, and startup and shutdown costs.
    data.load(filename=os.path.join(inputs_directory, "resource_unit_commitment_params.csv"),
              param=(resolve_model.min_stable_level_fraction,
                     resolve_model.ramp_rate_fraction,
                     resolve_model.min_down_time_hours,
                     resolve_model.min_up_time_hours,
                     resolve_model.unit_size_mw,
                     resolve_model.startup_cost_per_mw,
                     resolve_model.shutdown_cost_per_mw,
                     resolve_model.start_fuel_mmbtu_per_mw)
              )

    # The annual fixed cost of per unit of energy capacity ($/kWh-yr) for storage resources by vintage.
    # Indexed by NEW_BUILD_STORAGE_RESOURCES and VINTAGES in the model formulation
    data.load(filename=os.path.join(inputs_directory, "resource_vintage_storage_params.csv"),
              param=(resolve_model.energy_storage_cost_per_kwh_yr,
                     resolve_model.new_energy_capacity_fixed_o_and_m_dollars_per_kwh_yr)
              )

    # All resources modeled, with their technology, zone,
    # and flags for various operational and capacity characteristics.
    data.load(filename=os.path.join(inputs_directory, "resources.csv"),
              index=resolve_model.RESOURCES,
              param=(resolve_model.technology,
                     resolve_model.zone,
                     resolve_model.can_build_new,
                     resolve_model.capacity_limited,
                     resolve_model.can_retire,
                     resolve_model.thermal,
                     resolve_model.unit_commitment,
                     resolve_model.generate_at_max,
                     resolve_model.variable,
                     resolve_model.conventional_dr,
                     resolve_model.energy_efficiency,
                     resolve_model.storage,
                     resolve_model.increases_load,
                     resolve_model.load_only,
                     resolve_model.enable_hourly_energy_tracking
                     )
              )

    # Defines the relationship between each resource and reserve product.
    data.load(filename=os.path.join(inputs_directory, "reserve_resources.csv"),
              index=resolve_model.RESERVE_RESOURCES,
              param=(resolve_model.can_provide_spin,
                     resolve_model.can_provide_reg,
                     resolve_model.can_provide_lf_reserves,
                     resolve_model.contributes_to_min_gen,
                     resolve_model.contributes_to_freq_resp_total_req,
                     resolve_model.contributes_to_freq_resp_partial_req,
                     resolve_model.thermal_freq_response_fraction_of_commitment)
              )


    if resolve_model.itc_paired_storage:
        data.load(filename=os.path.join(inputs_directory, "resource_storage_paired.csv"),
                  index=resolve_model.PAIRED_STORAGE_RESOURCES,
                  param=resolve_model.paired_supply_resource)

    # The maximum capacity of each resource that can be built in each period.
    # Resources that do not have capacity limits enforced are not included.
    # Indexed by CAPACITY_LIMITED_RESOURCES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "capacity_limits.csv"),
              param=resolve_model.capacity_limit_mw
              )

    # Custom constraints for periods and timepoints
    # The first four columns of the file (group, index_1, index_2, attribute)
    # define the set CUSTOM_CONSTRAINT_PERIOD_GROUP_MAP or CUSTOM_CONSTRAINT_TIMEPOINT_GROUP_MAP
    # Frequently index_2 won't be needed - in this case it should be "None"
    if resolve_model.custom_period_constraints:
        print("...custom period constraints exist.")
        data.load(filename=os.path.join(inputs_directory, "custom_constraint_period_group_map.csv"),
                  index=resolve_model.CUSTOM_CONSTRAINT_PERIOD_GROUP_MAP,
                  param=resolve_model.custom_constraint_period_multiplier
                  )
    else:
        print("...no custom period constraints.")

    if resolve_model.custom_timepoint_constraints:
        print("...custom timepoint constraints exist.")
        data.load(filename=os.path.join(inputs_directory, "custom_constraint_timepoint_group_map.csv"),
                  index=resolve_model.CUSTOM_CONSTRAINT_TIMEPOINT_GROUP_MAP,
                  param=resolve_model.custom_constraint_timepoint_multiplier
                  )
    else:
        print("...no custom timepoint constraints.")

    # The planned installed capacity of each resource in each period.
    # The cost of planned capacity is assumed to be sunk and consequently is not included in the optimization,
    # but fixed operations and maintenance costs are included for planned capacity
    # Indexed by RESOURCES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "planned_installed_capacities.csv"),
              param=(resolve_model.planned_installed_capacity_mw,
                     resolve_model.min_operational_planned_capacity_mw,
                     resolve_model.planned_capacity_fixed_o_and_m_dollars_per_kw_yr)
              )

    # The minimum amount of new capacity of each resource that must be built through each period.
    # The cost of building these resources is not assumed to be sunk (in contrast to planned_installed_capacities.csv)
    # Indexed by NEW_BUILD_RESOURCES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "min_cumulative_new_build.csv"),
              param=resolve_model.min_cumulative_new_build_mw
              )

    # The annual capital and fixed operations and maintenence cost per unit of capacity ($/kW-yr)
    # for each resource that can install new capacity.  Costs can vary by installation year (vintage).
    # Indexed by NEW_BUILD_RESOURCES and VINTAGES in the model formulation
    data.load(filename=os.path.join(inputs_directory, "resource_vintage_params.csv"),
              param=(resolve_model.capital_cost_per_kw_yr,
                     resolve_model.new_capacity_fixed_o_and_m_dollars_per_kw_yr)
              )

    # The net qualifying capacity (NQC) fraction for firm and storage capacity resources.
    # Indexed by non-elcc resources and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "resource_reliability_nqc.csv"),
              param=resolve_model.net_qualifying_capacity_fraction
              )

    # The planned installed energy capacity of each storage resource in each period.
    # Indexed by STORAGE_RESOURCES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "planned_storage_energy_capacity.csv"),
              param=(resolve_model.planned_storage_energy_capacity_mwh,
                     resolve_model.planned_storage_energy_capacity_fixed_o_and_m_dollars_per_kwh_yr)
              )

    # The normalized profiles for each variable resource for each day and hour.
    # Indexed by VARIABLE_RESOURCES, DAYS, and HOURS_OF_DAY in the model formulation
    data.load(filename=os.path.join(inputs_directory, "shapes.csv"),
              param=(resolve_model.shape,
                     resolve_model.resource_downward_lf_req,
                     resolve_model.resource_upward_lf_req))

    # Limits on ramps for each ramp duration for each ramp-limited resource.
    # Indexed by RAMP_CONSTRAINED_RESOURCES and RAMP_DURATIONS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "ramps_intertimepoint.csv"),
              param=(resolve_model.ramp_up_intertimepoint_limit_fraction,
                     resolve_model.ramp_down_intertimepoint_limit_fraction)
              )

    # A range of single-value parameters including penalties for unserved energy, overgeneration,
    # and reserve violations; the durations of hydro and intertie ramps to constrain;
    # parameterizations of the sub-hourly behavior when providing regulation and load-following reserves;
    # parameterizations of the ability of variable generation to provide reserves;
    # whether to require renewable overbuild when satisfying RPS constraints;
    # whether to allow RPS banking;
    # the number of hours of duration that receives full ELCC credit;
    # and the assumed timeframe for operational reserves.
    data.load(filename=os.path.join(inputs_directory, "system_params.csv"),
              param=(resolve_model.unserved_energy_penalty_per_mw,
                     resolve_model.overgen_penalty_per_mw,
                     resolve_model.spin_violation_penalty_per_mw,
                     resolve_model.upward_reg_violation_penalty_per_mw,
                     resolve_model.downward_reg_violation_penalty_per_mw,
                     resolve_model.upward_lf_reserve_violation_penalty_per_mw,
                     resolve_model.downward_lf_reserve_violation_penalty_per_mw,
                     resolve_model.max_ramp_duration_to_constrain,
                     resolve_model.max_intertie_ramp_duration_to_constrain,
                     resolve_model.reg_dispatch_fraction,
                     resolve_model.lf_reserve_dispatch_fraction,
                     resolve_model.var_rnw_available_for_lf_reserves,
                     resolve_model.max_var_rnw_lf_reserves,
                     resolve_model.reserve_timeframe_fraction_of_hour,
                     resolve_model.allow_unserved_energy)
              )

    # The RPS target across all RPS zones by period.
    if resolve_model.include_RPS:
        data.load(filename = os.path.join(inputs_directory, "rps_policies_set.csv"),
                  index = resolve_model.RPS_POLICIES,
                  param = resolve_model.starting_rps_bank_mwh)

        data.load(filename = os.path.join(inputs_directory, "rps_policies_resources.csv"),
                  param= resolve_model.rps_eligible_fraction)

        data.load(filename = os.path.join(inputs_directory, "rps_policies_periods.csv"),
                  param=(resolve_model.rps_nonmodeled_mwh,
                         resolve_model.rps_unbundled_fraction_limit,
                         resolve_model.retail_sales_mwh,
                         resolve_model.rps_fraction_of_retail_sales)
                  )

        data.load(filename = os.path.join(inputs_directory, "rps_policies_zones.csv"),
                  param=resolve_model.is_zone_in_rps_policy
                  )

    # Map that relates transmission zones to newly buildable renewable resources.
    if resolve_model.deliverability:
        data.load(
            filename=os.path.join(inputs_directory, "resource_deliverability_map.csv"),
            set=resolve_model.RESOURCE_DELIVERABILITY_MAP
        )


    # The maximum amount of energy that can be dispatched (shed) annually from conventional demand response resources
    # Indexed by CONVENTIONAL_DR_RESOURCES and PERIODS in the model formulation
    data.load(filename=os.path.join(inputs_directory, "conventional_dr_period_limits.csv"),
              param=(resolve_model.conventional_dr_availability_hours_per_year,
                     resolve_model.conventional_dr_reliability_hours_per_day)
              )


    # The reliability margin target across all reliability zones in each period,
    # and other quantities related to the reliability margin.
    data.load(filename=os.path.join(inputs_directory, "reliability_constraint_resource_map.csv"),
              set=resolve_model.RELIABILITY_CONSTRAINT_RESOURCE_MAP
              )

    data.load(filename=os.path.join(inputs_directory, "reliability_constraints.csv"),
              param=(resolve_model.reliability_margin,
                     resolve_model.reliability_peak_load_mw,
                     resolve_model.reliability_capacity_adjustment_mw,
                     resolve_model.reliability_event_length_hour)
              )

    # Effective load carrying capability (ELCC) surface facet coefficients for wind and solar power.
    if resolve_model.allow_elcc_surfaces:
        data.load(filename=os.path.join(inputs_directory, "elcc_surface_axes.csv"),
                  index=resolve_model.ELCC_SURFACE_MAP,
                  param=(resolve_model.elcc_surface_axes,
                         resolve_model.elcc_surface_axes_map_multiplier)
                  )

        data.load(filename=os.path.join(inputs_directory, "elcc_surface_facets.csv"),
                  index=resolve_model.ELCC_SURFACE_FACETS_AXES_PERIOD,
                  param=resolve_model.elcc_surface_facets_axes_coefficient
                  )


    # Parameters for interday energy sharing group logic
    if resolve_model.interday_energy_sharing:
        data.load(filename=os.path.join(inputs_directory, "energy_sharing_interval_mapping.csv"),
                  index=resolve_model.DAYS,
                  param=resolve_model.interday_energy_sharing_interval_id
                  )

        data.load(filename=os.path.join(inputs_directory, "energy_sharing_max_to_move_within_group.csv"),
                  param=resolve_model.max_energy_to_share_hours
                  )

        data.load(filename=os.path.join(inputs_directory, "energy_sharing_daily_max_changes.csv"),
                  param=(resolve_model.daily_max_energy_budget_increase_hours,
                         resolve_model.daily_max_energy_budget_decrease_hours)
                  )

    # parameters for ee program
    if resolve_model.allow_ee_investment:
        data.load(filename=os.path.join(inputs_directory, "ee_params.csv"),
                  index=resolve_model.EE_PROGRAMS,
                  param=resolve_model.ee_t_and_d_losses_fraction
                  )

        data.load(filename=os.path.join(inputs_directory, "ee_period_params.csv"),
                  param=resolve_model.max_investment_in_period_aMW
                  )

        data.load(filename=os.path.join(inputs_directory, "ee_timepoint_params.csv"),
                  param=resolve_model.ee_shapes_btm_mwh_per_amw
                  )

    # parameters needed if you want to have existing resources generation being counted as using tx lines capacity
    if resolve_model.resource_use_tx_capacity:
        data.load(filename=os.path.join(inputs_directory, "resource_use_tx_capacity.csv"),
                  index=resolve_model.RESOURCE_TX_IDS,
                  param=(resolve_model.dedicated_import_resource,
                         resolve_model.tx_line_used,
                         resolve_model.resource_tx_direction)
                  )

    # parameters for semi-storage zones features where you can send and get back power to a "semi_storage_zone"
    # with hurdle rates
    if resolve_model.allow_semi_storage_zones:
        data.load(filename=os.path.join(inputs_directory, "semi_storage_zones_params.csv"),
                  index=resolve_model.SEMI_STORAGE_ZONES,
                  param=resolve_model.ssz_from_zone
                  )

    if resolve_model.allow_semi_storage_zones:
        data.load(filename=os.path.join(inputs_directory, "semi_storage_zones_period_params.csv"),
                  param=(resolve_model.ssz_max_flow_mw,
                         resolve_model.ssz_min_flow_mw,
                         resolve_model.ssz_positive_direction_hurdle_rate_per_mw,
                         resolve_model.ssz_negative_direction_hurdle_rate_per_mw)
                  )

    if resolve_model.allow_tx_build:
        data.load(filename=os.path.join(inputs_directory, "transmission_new_build_vintages.csv"),
                  param=(resolve_model.max_tx_build_mw,
                         resolve_model.min_tx_build_mw,
                         resolve_model.new_tx_fixed_cost_per_mw_yr)
                        )

    return data, flexible_params
