// Copyright (C) Stichting Deltares 2023. All rights reserved.
// 
// This file is part of the application DAM - Live.
// 
// DAM - Live is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// 
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// 
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// 
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Deltares.Dam.Data.CsvImporters;
using Deltares.Dam.Data.DataPlugins.Configuration;
using Deltares.Dam.Data.Importers;
using Deltares.Dam.Data.IO;
using Deltares.Geometry;
using Deltares.Geotechnics.IO;
using Deltares.Geotechnics.Soils;
using Deltares.Geotechnics.SurfaceLines;
using Deltares.Maps;
using Deltares.Standard;
using Deltares.Standard.Logging;
using GeoAPI.Geometries;
using ProgressDelegate = Deltares.DamEngine.Data.Standard.Calculation.ProgressDelegate;

namespace Deltares.Dam.Data.DataPlugins;
// TODO: Import PL1-lines 

/// <summary>
/// ImportMode
/// </summary>
internal enum ImportMode
{
    DikeRingAndLocationData,
    All
}

/// <summary>
/// Exception for DataPluginImporter 
/// </summary>
public class DataPluginImporterException : ApplicationException
{
    public DataPluginImporterException() {}

    public DataPluginImporterException(string message)
        : base(message) {}

    public DataPluginImporterException(string message, Exception inner)
        : base(message, inner) {}
}

/// <summary>
/// DataPlugin implementation to import from files
/// </summary>
public class DataPluginImporter : IDataPlugin
{
    private readonly IFeatureRepository repository = new FeatureRepository();
    private bool isAllDataImported;
    private bool isDikeRingAndLocationDataImported;
    private WaterBoard waterBoard;
    private List<DataAttribute> attributes;
    private DamProjectType damProjectType;

    private string damProjectFolder;

    /// <summary>
    /// Initializes a new instance of the <see cref="DataPluginImporter"/> class.
    /// </summary>
    public DataPluginImporter()
    {
        InitializeWaterBoard();
        attributes = new List<DataAttribute>();
    }

    /// <summary>
    /// Gets or sets the attributes.
    /// </summary>
    /// <value>
    /// The attributes.
    /// </value>
    /// <exception cref="System.ArgumentNullException">value</exception>
    public IEnumerable<DataAttribute> Attributes
    {
        get
        {
            return attributes;
        }
        set
        {
            if (value == null)
            {
                throw new ArgumentNullException("value");
            }

            isAllDataImported = false;
            attributes = value as List<DataAttribute> ?? value.ToList();
        }
    }

    /// <summary>
    /// Sets the type of the dam project.
    /// </summary>
    /// <value>
    /// The type of the dam project.
    /// </value>
    public DamProjectType DamProjectType
    {
        set
        {
            damProjectType = value;
        }
    }

    /// <summary>
    /// Gets the import log messages.
    /// </summary>
    /// <value>
    /// The import log messages.
    /// </value>
    public List<LogMessage> ImportLogMessages { get; } = new List<LogMessage>();

    /// <summary>
    /// Gets or sets the dam project folder.
    /// </summary>
    /// <value>
    /// The dam project folder.
    /// </value>
    public string DamProjectFolder
    {
        get
        {
            return damProjectFolder;
        }
        set
        {
            if (damProjectFolder != value)
            {
                damProjectFolder = value;
                Clear();
            }
        }
    }

    /// <summary>
    /// Gets the data folder.
    /// </summary>
    /// <value>
    /// The data folder.
    /// </value>
    public string DataFolder { get; private set; }

    /// <summary>
    /// Get data source definitions
    /// </summary>
    /// <returns></returns>
    public IEnumerable<DataSource> DataSources { get; private set; }

    /// <summary>
    /// Get info for WaterBoard
    /// </summary>
    /// <returns></returns>
    public Info WaterBoardInfo
    {
        get
        {
            ThrowIfDataNotRead();
            return new Info
            {
                Name = waterBoard.Name,
                Description = waterBoard.Description,
                Source = DataSourceManager.GetSource(waterBoard, "Name")
            };
        }
    }

    /// <summary>
    /// Gets the map geometry identifier list.
    /// </summary>
    /// <value>
    /// The map geometry identifier list.
    /// </value>
    public IEnumerable<string> MapGeometryIdList
    {
        get
        {
            ThrowIfDataNotRead();
            return repository.FeatureIds.Select(id => id.ToString());
        }
    }

    /// <summary>
    /// Returns a <see cref="System.String" /> that represents this instance.
    /// </summary>
    /// <returns>
    /// A <see cref="System.String" /> that represents this instance.
    /// </returns>
    public override string ToString()
    {
        return "DataPluginImporter";
    }

    /// <summary>
    /// Define data sources
    /// </summary>
    /// <param name="dataFolder"> </param>
    /// <param name="dataSources"></param>
    public void SetDataSources(string dataFolder, IEnumerable<DataSource> dataSources)
    {
        isAllDataImported = false;
        DataFolder = dataFolder;
        DataSources = dataSources;
    }

    /// <summary>
    /// Gets the surface line identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetSurfaceLineIdList(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        return dike.SurfaceLines2.Select(surfaceLine => surfaceLine.Name);
    }

    /// <summary>
    /// Gets the dike ring identifier list.
    /// </summary>
    /// <returns></returns>
    public IEnumerable<string> GetDikeRingIdList()
    {
        ImportDikeRingAndLocationData();
        return waterBoard.Dikes.Select(dike => dike.Name);
    }

    /// <summary>
    /// Imports the data for dike rings.
    /// </summary>
    /// <param name="dikeRingIds">The dike ring ids.</param>
    /// <param name="progress">The progress.</param>
    public void ImportDataForDikeRings(IEnumerable<string> dikeRingIds,
                                       ProgressDelegate progress)
    {
        isAllDataImported = false;
        isDikeRingAndLocationDataImported = false;
        ImportAllData(dikeRingIds, progress);
    }

    /// <summary>
    /// Gets the scenario list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetScenarioList(string dikeRingId, string locationId)
    {
        Location location = GetLocation(dikeRingId, locationId);
        return location.Scenarios.Select(scenario => scenario.LocationScenarioID);
    }

    /// <summary>
    /// Gets the dike ring information.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public Info GetDikeRingInfo(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        return new Info
        {
            Name = dike.Name,
            Description = dike.Description,
            Source = DataSourceManager.GetSource(dike, "Name")
        };
    }

    /// <summary>
    /// Gets the dike parameters.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetDikeParameters(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        Dictionary<string, string> nameValuePairs = dike.GetParametersAsNameValuePairs();

        return GetParametersFromDictionary(nameValuePairs, dike);
    }

    /// <summary>
    /// Gets the location identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetLocationIdList(string dikeRingId)
    {
        ImportDikeRingAndLocationData();
        Dike dike = GetDike(dikeRingId);
        return dike.Locations.Select(location => location.Name);
    }

    /// <summary>
    /// Gets the location information.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <returns></returns>
    public Info GetLocationInfo(string dikeRingId, string locationId)
    {
        Location location = GetLocation(dikeRingId, locationId);
        return new Info
        {
            Name = location.Name,
            Description = "",
            Source = DataSourceManager.GetSource(location, "Name")
        };
    }

    /// <summary>
    /// Gets the location details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetLocationDetails(string dikeRingId, string locationId)
    {
        Location location = GetLocation(dikeRingId, locationId);

        Dictionary<string, string> nameValuePairs = location.GetParametersAsNameValuePairs();
        return GetParametersFromDictionary(nameValuePairs, location);
    }

    /// <summary>
    /// Gets the scenario details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <param name="scenarioId">The scenario identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetScenarioDetails(string dikeRingId, string locationId, string scenarioId)
    {
        Location location = GetLocation(dikeRingId, locationId);
        Scenario scenario = GetScenario(dikeRingId, locationId, scenarioId);

        if (scenario == null)
        {
            yield break;
        }

        Dictionary<string, string> nameValuePairs = scenario.GetParametersAsNameValuePairs();
        foreach (KeyValuePair<string, string> nameValuePair in nameValuePairs)
        {
            yield return new NameValueParameter
            {
                ParameterName = nameValuePair.Key,
                ParameterValue = nameValuePair.Value,
                Source = DataSourceManager.GetSource(location, nameValuePair.Key)
            };
        }
    }

    public void DisposeImportedItem(string dikeRingId)
    {
        GetDike(dikeRingId).Dispose();
    }

    /// <summary>
    /// Gets the surface line points.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="surfaceLineId">The surface line identifier.</param>
    /// <returns></returns>
    public IList<GeometryPoint> GetSurfaceLinePoints(string dikeRingId, string surfaceLineId)
    {
        SurfaceLine2 surfaceLine = GetSurfaceLine(dikeRingId, surfaceLineId);

        var surfaceLinePoints = new List<GeometryPoint>();
        for (var pointIndex = 0; pointIndex < surfaceLine.Geometry.Points.Count(); pointIndex++)
        {
            surfaceLinePoints.Add(surfaceLine.Geometry.Points[pointIndex]);
        }

        return surfaceLinePoints;
    }

    /// <summary>
    /// Gets the surface line characteristic points.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="surfaceLineId">The surface line identifier.</param>
    /// <returns></returns>
    public IEnumerable<DpCharacteristicPoint> GetSurfaceLineCharacteristicPoints(string dikeRingId,
                                                                                 string surfaceLineId)
    {
        SurfaceLine2 surfaceLine = GetSurfaceLine(dikeRingId, surfaceLineId);

        foreach (CharacteristicPointType type in Enum.GetValues(typeof(CharacteristicPointType)))
        {
            // get & check point
            GeometryPoint point = surfaceLine.CharacteristicPoints.GetGeometryPoint(type);
            if (point == null)
            {
                continue;
            }

            if ((point.X == -1.0) && (point.Y == -1.0) && (point.Z == -1.0))
            {
                continue;
            }

            // create point
            yield return new DpCharacteristicPoint
            {
                Id = type.ToString(),
                X = point.X,
                Y = point.Y,
                Z = point.Z
            };
        }
    }

    /// <summary>
    /// Gets the segment identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetSegmentIdList(string dikeRingId)
    {
        ThrowIfDataNotRead();
        return waterBoard.Segments.Select(segment => segment.Name);
    }

    /// <summary>
    /// Gets the profile1 d identifier list for segment.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="segmentId">The segment identifier.</param>
    /// <param name="segmentFailureMechanismType">Type of the segment failure mechanism.</param>
    /// <returns></returns>
    public IEnumerable<string> GetProfile1DIdListForSegment(string dikeRingId, string segmentId, FailureMechanismSystemType segmentFailureMechanismType)
    {
        ThrowIfDataNotRead();
        Segment segment = GetSegment(segmentId);
        foreach (SoilGeometryProbability soilGeometryProbability in segment.SoilProfileProbabilities)
        {
            if (soilGeometryProbability.SoilProfile != null)
            {
                if ((soilGeometryProbability.SegmentFailureMechanismType == null) || (soilGeometryProbability.SegmentFailureMechanismType.Value == segmentFailureMechanismType))
                {
                    yield return soilGeometryProbability.SoilGeometryName;
                }
            }
        }
    }

    /// <summary>
    /// Gets the profile2 d identifier list for segment.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="segmentId">The segment identifier.</param>
    /// <param name="segmentFailureMechanismType">Type of the segment failure mechanism.</param>
    /// <returns></returns>
    public IEnumerable<string> GetProfile2DIdListForSegment(string dikeRingId, string segmentId, FailureMechanismSystemType segmentFailureMechanismType)
    {
        ThrowIfDataNotRead();
        Segment segment = GetSegment(segmentId);
        foreach (SoilGeometryProbability soilGeometryProbability in segment.SoilProfileProbabilities)
        {
            if (soilGeometryProbability.SoilProfile == null)
            {
                if ((soilGeometryProbability.SegmentFailureMechanismType == null) || (soilGeometryProbability.SegmentFailureMechanismType.Value == segmentFailureMechanismType))
                {
                    yield return soilGeometryProbability.SoilGeometryName;
                }
            }
        }
    }

    /// <summary>
    /// Gets the segment profile1 d details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="segmentId">The segment identifier.</param>
    /// <param name="profile1DId">The profile1 d identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetSegmentProfile1DDetails(string dikeRingId, string segmentId,
                                                                      string profile1DId, FailureMechanismSystemType segmentFailureMechanismType)
    {
        Segment segment = GetSegment(segmentId);
        Dictionary<string, string> nameValuePairs = segment.GetParametersForSoilProfile1DAsNameValuePairs(profile1DId, segmentFailureMechanismType);

        return GetParametersFromDictionary(nameValuePairs, segment);
    }

    /// <summary>
    /// Gets the segment profile2 d details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="segmentId">The segment identifier.</param>
    /// <param name="profile2DId">The profile2 d identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetSegmentProfile2DDetails(string dikeRingId, string segmentId,
                                                                      string profile2DId, FailureMechanismSystemType failureMechanismSystemType)
    {
        Segment segment = GetSegment(segmentId);
        Dictionary<string, string> nameValuePairs = segment.GetParametersForSoilProfile2DAsNameValuePairs(profile2DId, failureMechanismSystemType);

        return GetParametersFromDictionary(nameValuePairs, segment);
    }

    /// <summary>
    /// Gets the soil profile1 d identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetSoilProfile1DIdList(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        return dike.SoilProfiles.Select(soilProfile => soilProfile.Name);
    }

    /// <summary>
    /// Gets the soil profile1 d details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="soilProfile1DId">The soil profile1 d identifier.</param>
    /// <returns></returns>
    public DpSoilProfile GetSoilProfile1DDetails(string dikeRingId, string soilProfile1DId)
    {
        SoilProfile1D soilProfile = GetSoilProfile(dikeRingId, soilProfile1DId);
        var dpSoilProfile = new DpSoilProfile
        {
            Layers = new List<DpLayer>()
        };
        for (var layerIndex = 0; layerIndex < soilProfile.Layers.Count; layerIndex++)
        {
            var dpLayer = new DpLayer
            {
                TopLevel = soilProfile.Layers[layerIndex].TopLevel,
                SoilName = soilProfile.Layers[layerIndex].Soil.Name
            };
            dpSoilProfile.Layers.Add(dpLayer);
        }

        var bottomLevelLayer = new DpLayer
        {
            TopLevel = soilProfile.BottomLevel,
            SoilName = SoilProfile1D.SoilProfileBottomLevelId
        };
        dpSoilProfile.Layers.Add(bottomLevelLayer);
        return dpSoilProfile;
    }

    /// <summary>
    /// Gets the soil profile2 d identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    /// <exception cref="System.NotImplementedException"></exception>
    public IEnumerable<string> GetSoilProfile2DIdList(string dikeRingId)
    {
        ThrowIfDataNotRead();
        throw new NotImplementedException();
    }

    /// <summary>
    /// Gets the soil profile2 d details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="soilProfile2DId">The soil profile2 d identifier.</param>
    /// <returns></returns>
    /// <exception cref="System.NotImplementedException"></exception>
    public string GetSoilProfile2DDetails(string dikeRingId, string soilProfile2DId)
    {
        ThrowIfDataNotRead();
        throw new NotImplementedException();
    }

    /// <summary>
    /// Gets the soil identifier list.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<string> GetSoilIdList(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        IEnumerable<string> soilNames = dike.SoilList.Soils.Select(soil => soil.Name).ToList();
        return soilNames;
    }

    /// <summary>
    /// Gets the soil aquifer values.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetSoilAquiferValues(string dikeRingId)
    {
        Dike dike = GetDike(dikeRingId);
        var res = new List<NameValueParameter>();
        foreach (Soil soil in dike.SoilList.Soils)
        {
            if (dike.SoilList.AquiferDictionary.ContainsKey(soil))
            {
                var nvp = new NameValueParameter();
                nvp.ParameterName = soil.Name;
                nvp.ParameterValue = dike.SoilList.AquiferDictionary[soil].ToString();
                nvp.Source = DataSourceSystemType.MSoilBase;
                res.Add(nvp);
            }
        }

        return res;
    }

    /// <summary>
    /// Gets the soil details.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="soilId">The soil identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetSoilDetails(string dikeRingId, string soilId)
    {
        Soil soil = GetSoil(dikeRingId, soilId);
        Dictionary<string, string> nameValuePairs = SoilUtils.GetParametersAsNameValuePairs(soil);

        return GetParametersFromDictionary(nameValuePairs, soil);
    }

    /// <summary>
    /// Gets the map geometry.
    /// </summary>
    /// <param name="mapGeometryId">The map geometry identifier.</param>
    /// <returns></returns>
    public string GetMapGeometry(string mapGeometryId)
    {
        ThrowIfDataNotRead();
        IGeometry geom = GetGeometryById(mapGeometryId);
        return geom.AsText();
    }

    /// <summary>
    /// Gets the map geometry attributes.
    /// </summary>
    /// <param name="mapGeometryId">The map geometry identifier.</param>
    /// <returns></returns>
    public IEnumerable<NameValueParameter> GetMapGeometryAttributes(string mapGeometryId)
    {
        ThrowIfDataNotRead();
        IGeometry geom = GetGeometryById(mapGeometryId);

        IEnumerable<KeyValuePair<string, object>> geomAttributes = repository.GetAttributes(geom);

        return geomAttributes
               .Select(kvp =>
                           new NameValueParameter
                           {
                               ParameterName = kvp.Key,
                               ParameterValue = kvp.Value.ToString()
                           }).ToList();
    }

    private void InitializeWaterBoard()
    {
        if (waterBoard != null)
        {
            waterBoard.Dispose();
        }

        waterBoard = new WaterBoard();
    }

    /// <summary>
    /// Clears this instance.
    /// </summary>
    private void Clear()
    {
        InitializeWaterBoard();

        isAllDataImported = false;
        isDikeRingAndLocationDataImported = false;
    }

    /// <summary>
    /// Gets the dike.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    private Dike GetDike(string dikeRingId)
    {
        ThrowIfDataNotRead();
        return waterBoard.Dikes.First(d => d.Name.Equals(dikeRingId));
    }

    /// <summary>
    /// Gets the or create dike.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <returns></returns>
    private Dike GetOrCreateDike(string dikeRingId)
    {
        // first check to see if this dike is already known, if not create it
        Dike dike = null;

        // try to get it
        if (waterBoard.Dikes.Any())
        {
            string id = dikeRingId;
            dike = waterBoard.Dikes.FirstOrDefault(x => x.Name.Equals(id));
        }

        // not found, create it
        if (dike == null)
        {
            dike = new Dike
            {
                Name = dikeRingId
            };
            waterBoard.Dikes.Add(dike);
        }

        return dike;
    }

    /// <summary>
    /// Gets the segment.
    /// </summary>
    /// <param name="segmentID">The segment identifier.</param>
    /// <returns></returns>
    private Segment GetSegment(string segmentID)
    {
        ThrowIfDataNotRead();
        return waterBoard.Segments.First(s => s.Name.Equals(segmentID));
    }

    /// <summary>
    /// Gets the location.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <returns></returns>
    private Location GetLocation(string dikeRingId, string locationId)
    {
        Dike dike = GetDike(dikeRingId);
        return dike.Locations.First(l => l.Name.Equals(locationId));
    }

    /// <summary>
    /// Gets a soil.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="soilId">The soil identifier.</param>
    /// <returns></returns>
    private Soil GetSoil(string dikeRingId, string soilId)
    {
        Dike dike = GetDike(dikeRingId);
        Soil soil = dike.SoilList.GetSoilByName(soilId);
        return soil;
    }

    /// <summary>
    /// Gets a soil profile.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="soilProfile1DId">The soil profile1 d identifier.</param>
    /// <returns></returns>
    private SoilProfile1D GetSoilProfile(string dikeRingId, string soilProfile1DId)
    {
        Dike dike = GetDike(dikeRingId);
        return dike.SoilProfiles.First(p => p.Name.Equals(soilProfile1DId));
    }

    /// <summary>
    /// Gets a scenario.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="locationId">The location identifier.</param>
    /// <param name="scenarioId">The scenario identifier.</param>
    /// <returns></returns>
    private Scenario GetScenario(string dikeRingId, string locationId, string scenarioId)
    {
        Dike dike = GetDike(dikeRingId);
        Location location = dike.Locations.FirstOrDefault(l => l.Name.Equals(locationId));
        return (location == null ? null : location.Scenarios.FirstOrDefault(s => s.LocationScenarioID.Equals(scenarioId)));
    }

    /// <summary>
    /// Gets a surface line.
    /// </summary>
    /// <param name="dikeRingId">The dike ring identifier.</param>
    /// <param name="surfaceLineId">The surface line identifier.</param>
    /// <returns></returns>
    private SurfaceLine2 GetSurfaceLine(string dikeRingId, string surfaceLineId)
    {
        Dike dike = GetDike(dikeRingId);
        return dike.SurfaceLines2.First(x => x.Name.Equals(surfaceLineId));
    }

    /// <summary>
    /// Gets the parameters from dictionary.
    /// </summary>
    /// <param name="dictionary">The dictionary.</param>
    /// <param name="source">The source.</param>
    /// <returns></returns>
    private IEnumerable<NameValueParameter> GetParametersFromDictionary(
        Dictionary<string, string> dictionary, object source)
    {
        IEnumerable<NameValueParameter> pairs = dictionary.Select(nameValuePair => new NameValueParameter
        {
            ParameterName = nameValuePair.Key,
            ParameterValue = nameValuePair.Value,
            Source = DataSourceManager.GetSource(source, nameValuePair.Key)
        });

        return new List<NameValueParameter>(pairs);
    }

    /// <summary>
    /// Import all csv files
    /// </summary>
    /// <param name="importFolder">The import folder.</param>
    /// <param name="importMode">The import mode.</param>
    /// <param name="progress">The progress.</param>
    /// <param name="csvImporter">The CSV importer.</param>
    /// <returns></returns>
    private bool ImportCsvFiles(string importFolder,
                                ImportMode importMode, ProgressDelegate progress, ref CsvImporter csvImporter)
    {
        DataSourceManager.CurrentSource = DataSourceSystemType.Csv;
        bool isImportOnlyLocations = (importMode == ImportMode.DikeRingAndLocationData);
        csvImporter.ImportCsvDataFromDirectory(importFolder, isImportOnlyLocations, progress, damProjectType);

        // add messages
        List<LogMessage> messages = csvImporter.ErrorMessages;
        ImportLogMessages.AddRange(messages);

        DataSourceManager.CurrentSource = DataSourceSystemType.User;

        // check for fatal messages
        bool importOk = messages.All(m => m.MessageType != LogMessageType.FatalError);

        return importOk;
    }

    /// <summary>
    /// Import name of the specified soilbase to all dikerings with id as specified in dikeRingIds.
    /// Note that it does NOT import the soils in the database themselves.
    /// If dikeringId not found then a new dike is created for that dikeRingId.
    /// </summary>
    /// <param name="importFolder">path where the data can be found</param>
    /// <param name="dikeRingIds"></param>
    private void ImportMSoilBaseName(string importFolder, IEnumerable<string> dikeRingIds)
    {
        if (dikeRingIds == null)
        {
            return;
        }

        DataSourceManager.CurrentSource = DataSourceSystemType.MSoilBase;

        foreach (string dikeRingId in dikeRingIds)
        {
            Dike dike = GetOrCreateDike(dikeRingId);

            dike.SoilDatabaseName = importFolder;
        }

        DataSourceManager.CurrentSource = DataSourceSystemType.User;
    }

    /// <summary>
    /// Import the specified MGeobase to all dikerings with id as specified in dikeRingIds
    /// Note that this does import the 1D soil profiles together with the soils and their parameters.
    /// If dikeringId not found then a new dike is created for that dikeRingId
    /// </summary>
    /// <param name="importFolder">path where the data can be found</param>
    /// <param name="dikeRingIds"></param>
    private void ImportMGeobase(string importFolder, IEnumerable<string> dikeRingIds)
    {
        if (dikeRingIds == null)
        {
            return;
        }

        DataSourceManager.CurrentSource = DataSourceSystemType.MGeobase;

        foreach (string dikeRingId in dikeRingIds)
        {
            // first check to see if this dike is already known, if not create it
            Dike dike = GetOrCreateDike(dikeRingId);

            dike.SoilDatabaseName = importFolder;
            dike.AddSoilProfilesFromDB();
        }

        DataSourceManager.CurrentSource = DataSourceSystemType.User;
    }

    /// <summary>
    /// Import data from Iris shapefiles
    /// </summary>
    /// <param name="importFolder">path where the data can be found</param>
    private void ImportIrisFiles(string importFolder)
    {
        DataSourceManager.CurrentSource = DataSourceSystemType.Iris;

        // get locations (locationid and beginpoint and endpoint in RD) from "locations.shp" (LocationsNew)
        // get line of buitenkruinlijn
        // voeg aan elke locatie snijpunt met buitenkruinlijn toe (geen snijpunt is exception)
        // nu heeft locatie een begin-, eind- en buitenrkuinpunt
        // get line of binnenkruinlijn
        // voeg aan elke locatie snijpunt met binnenkruinlijn toe (geen snijpunt is exception)
        // nu heeft locatie een begin-, eind-, binnen- en buitenrkuinpunt

        // er is al een lijst met locaties in Waterboard WaterBoard.Dike[0].Locations (LocationsOrg)
        // LocationOrg.GeometryPoint[DikeTopAtRiver] = LocationNew.buitenkuinpunt
        // LocationOrg.GeometryPoint[DikeTopAtPolder] = LocationNew.binnnenkuinpunt
        //  Als LocationNew.id niet gevonden kan worden in LocationsOrg dan exception

        // Get all locations from waterboard            
        IEnumerable<Location> locations = GetAllLocations();

        var importer = new LocationCharacteristicPointImporter(importFolder, locations);
        importer.Import();

        DataSourceManager.CurrentSource = DataSourceSystemType.User;
    }

    /// <summary>
    /// Get background shapefile for this waterboard
    /// </summary>
    /// <param name="importFolder">path where the data can be found</param>
    private void ImportBackgroundShapeFiles(string importFolder)
    {
        DataSourceManager.CurrentSource = DataSourceSystemType.Gis;

        var reader = new ShapeFileReader(importFolder)
        {
            IgnoreAttributeData = true
        };
        var importer = new FeatureImporter(repository, reader);
        importer.Import();

        DataSourceManager.CurrentSource = DataSourceSystemType.User;
    }

    /// <summary>
    /// Get all locations for all dikes
    /// </summary>
    /// <returns></returns>
    private IEnumerable<Location> GetAllLocations()
    {
        return waterBoard.Dikes.SelectMany(dike => dike.Locations);
    }

    /// <summary>
    /// Pre conditions:
    /// - A list of waterboard locations with initialized x,y values
    /// - shape file ccontaining the attributes and data
    /// 
    /// Post condition:
    /// - Location properties (in relation to the attributes) are set
    /// </summary>
    /// <param name="importFolder"></param>
    /// <param name="dataAttributes"></param>
    /// <param name="importMode"></param>
    /// <param name="dikeRingIds"></param>
    /// <param name="progress"></param>
    private void ImportDataShapeFiles(string importFolder, IEnumerable<DataAttribute> dataAttributes,
                                      ImportMode importMode, IEnumerable<string> dikeRingIds,
                                      ProgressDelegate progress,
                                      ref List<CsvImporterLocations.LocationRecord> locationRecords)
    {
        DataSourceManager.CurrentSource = DataSourceSystemType.Gis;

        if (importFolder == null)
        {
            throw new ArgumentNullException("importFolder");
        }

        var importErrorLogFile = @"shapefiletrace.log";
        if (!string.IsNullOrEmpty(DamProjectFolder))
        {
            importErrorLogFile = Path.Combine(DamProjectFolder, importErrorLogFile);
        }

        List<DataAttribute> dataAttributesList = dataAttributes as List<DataAttribute> ?? dataAttributes.ToList();
        var exceptions = new List<Exception>();

        var locationImporter = LocationShapeFileImporter.Create(
            locationRecords, dataAttributesList, importFolder);
        if (locationImporter != null)
        {
            locationImporter.Import();
            locationRecords = locationImporter.ImportedItems.ToList();
        }

        // Skip reading properties when only dikering and locationdata is required
        if (importMode == ImportMode.All)
        {
            //
            // Location Property Importer
            // This is where all the attribute values are read form the shapefiles
            //
            var importer = new LocationPropertyImporter(importFolder, dataAttributesList)
            {
                Targets = locationRecords
            };

            importer.Import(progress);
            exceptions.AddRange(importer.ImportErrors);
            foreach (string importerMessage in importer.ImproperAttributeMessages)
            {
                var lm = new LogMessage(LogMessageType.Error, this, importerMessage);
                ImportLogMessages.Add(lm);
            }
        }

        using (var sw = new StreamWriter(importErrorLogFile, false))
        {
            if (!exceptions.Any())
            {
                return;
            }

            foreach (Exception error in exceptions)
            {
                //defaultWriter.Write(error.Message);
                sw.WriteLine("Date: {0}, Error type: {1},  Message: {2}",
                             DateTime.Now.ToShortDateString(), error.GetType().Name, error.Message);
                var lm = new LogMessage(LogMessageType.Error, this, error.Message);
                ImportLogMessages.Add(lm);
            }
        }
    }

    /// <summary>
    /// Gets the geometry by identifier.
    /// </summary>
    /// <param name="mapGeometryId">The map geometry identifier.</param>
    /// <returns></returns>
    /// <exception cref="System.ArgumentNullException">mapGeometryId</exception>
    /// <exception cref="System.ArgumentException">Can't find a geometry with this id</exception>
    private IGeometry GetGeometryById(string mapGeometryId)
    {
        if (mapGeometryId == null)
        {
            throw new ArgumentNullException("mapGeometryId");
        }

        var id = new Guid(mapGeometryId);

        IGeometry geom = repository.GetGeometryById(id);
        if (geom == null)
        {
            throw new ArgumentException("Can't find a geometry with this id");
        }

        return geom;
    }

    /// <summary>
    /// Import only Dikering and location id's
    /// </summary>
    private void ImportDikeRingAndLocationData()
    {
        if (isDikeRingAndLocationDataImported || isAllDataImported)
        {
            return;
        }

        var csvImporter = new CsvImporter();
        foreach (DataSource dataSource in DataSources)
        {
            ImportDikeRingAndLocationData(dataSource, ref csvImporter);
        }

        var combineImportedData = new CombineImportedData
        {
            WaterBoard = waterBoard,
            LocationRecords = csvImporter.LocationRecords
        };
        combineImportedData.AddCsvDataToDikes();
        isDikeRingAndLocationDataImported = true;
    }

    /// <summary>
    /// Gets the data location.
    /// </summary>
    /// <param name="dataSource">The data source.</param>
    /// <returns></returns>
    private string GetDataLocation(DataSource dataSource)
    {
        string dataLocation = dataSource.DataLocation;
        if (!string.IsNullOrEmpty(DataFolder)
            && (!Directory.Exists(dataLocation) || !File.Exists(dataLocation)))
        {
            dataLocation = Path.Combine(DataFolder, dataLocation);
        }

        return dataLocation;
    }

    /// <summary>
    /// Imports the dike ring and location data.
    /// </summary>
    /// <param name="dataSource">The data source.</param>
    /// <param name="csvImporter"></param>
    /// <exception cref="System.ArgumentOutOfRangeException"></exception>
    private void ImportDikeRingAndLocationData(DataSource dataSource, ref CsvImporter csvImporter)
    {
        string dataLocation = GetDataLocation(dataSource);

        switch (dataSource.DataSourceType)
        {
            case DataSourceType.CsvFiles:
                // New importer
                ImportCsvFiles(dataLocation, ImportMode.DikeRingAndLocationData, null, ref csvImporter);
                break;
            case DataSourceType.DataShapeFiles:
                List<CsvImporterLocations.LocationRecord> locationRecords = csvImporter.LocationRecords;
                ImportShapeFile(dataLocation, ref locationRecords);
                csvImporter.LocationRecords = locationRecords;
                break;
            case DataSourceType.Iris:
            case DataSourceType.BackgroundShapeFiles:
            case DataSourceType.MSoilBase:
            case DataSourceType.MGeobase:
                break;
            default:
                throw new ArgumentOutOfRangeException();
        }
    }

    /// <summary>
    /// Imports the data.
    /// </summary>
    /// <param name="dataSource">The data source.</param>
    /// <param name="dikeRingIdList">The dike ring identifier list.</param>
    /// <param name="importMode">The import mode.</param>
    /// <param name="progress">The progress.</param>
    /// <param name="csvImporter">The CSV importer.</param>
    /// <exception cref="System.ArgumentOutOfRangeException"></exception>
    private void ImportData2(DataSource dataSource,
                             IEnumerable<string> dikeRingIdList, ImportMode importMode,
                             ProgressDelegate progress, ref CsvImporter csvImporter)
    {
        string dataLocation = GetDataLocation(dataSource);
        switch (dataSource.DataSourceType)
        {
            case DataSourceType.CsvFiles:
                // New importer
                ImportCsvFiles(dataLocation, importMode, progress, ref csvImporter);
                break;
            case DataSourceType.Iris:
                ImportIrisFiles(dataLocation);
                break;
            case DataSourceType.DataShapeFiles:
                List<CsvImporterLocations.LocationRecord> locationRecords = csvImporter.LocationRecords;
                ImportDataShapeFiles(dataLocation, attributes, importMode, dikeRingIdList, progress, ref locationRecords);
                csvImporter.LocationRecords = locationRecords;
                break;
            case DataSourceType.BackgroundShapeFiles:
                if (importMode == ImportMode.All)
                {
                    ImportBackgroundShapeFiles(dataLocation);
                }

                break;
            case DataSourceType.MSoilBase:
                ImportMSoilBaseName(dataLocation, dikeRingIdList);
                break;
            case DataSourceType.MGeobase:
                ImportMGeobase(dataLocation, dikeRingIdList);
                break;
            default:
                throw new ArgumentOutOfRangeException();
        }
    }

    /// <summary>
    /// Imports the shape file.
    /// </summary>
    /// <param name="dataLocation">The data location.</param>
    /// <param name="locationRecords">The location records.</param>
    private void ImportShapeFile(string dataLocation, ref List<CsvImporterLocations.LocationRecord> locationRecords)
    {
        // get locations
        DataAttribute configuredAttribute =
            attributes.SingleOrDefault(a => a.AttributeId.Equals(
                                           LocationShapeFileAttributeMap.LocationAttributeId,
                                           StringComparison.InvariantCultureIgnoreCase));
        if (configuredAttribute == null)
        {
            return;
        }

        var locationImporter = LocationShapeFileImporter.Create(
            locationRecords, attributes, dataLocation);
        if (locationImporter != null)
        {
            locationImporter.Import();
            locationRecords = locationImporter.ImportedItems.ToList();
        }
    }

    /// <summary>
    /// Imports all data.
    /// </summary>
    /// <param name="dikeRingIds">The dike ring ids.</param>
    /// <param name="progress">The progress.</param>
    private void ImportAllData(IEnumerable<string> dikeRingIds, ProgressDelegate progress)
    {
        // do nothing if all data is already imported
        if (isAllDataImported)
        {
            return;
        }

        ImportData(ImportMode.All, dikeRingIds, progress);

        isAllDataImported = true;
    }

    /// <summary>
    /// Import all data
    /// </summary>
    private void ImportData(ImportMode importMode, IEnumerable<string> dikeRingIds, ProgressDelegate progress)
    {
        ThrowIfNoDataDefinitions();
        if (!isDikeRingAndLocationDataImported)
        {
            // If locations and dikeringIds were not read yet then we start with a new waterboard
            // Else continue with waterboard that was created when importing the locations
            InitializeWaterBoard();
        }

        waterBoard.Description = "Waterschap";
        var csvImporter = new CsvImporter();

        List<string> dikeRingIdList = dikeRingIds == null ? null : dikeRingIds as List<string> ?? dikeRingIds.ToList();

        foreach (DataSource dataSource in DataSources)
        {
            // Skip Iris because it needs the locations
            // Data from shapefiles is added to locationrecords from csvImporter
            if (dataSource.DataSourceType != DataSourceType.Iris)
            {
                ImportData2(dataSource, dikeRingIdList, importMode, progress, ref csvImporter);
            }
        }

        // create and fill the locations, scenarios etc. with the data from csv and shapefiles
        AddCsvDataToDikes(csvImporter);
        // Now that the locations are defined, read Iris data
        foreach (DataSource dataSource in DataSources)
        {
            if (dataSource.DataSourceType == DataSourceType.Iris)
            {
                ImportData2(dataSource, dikeRingIdList, importMode, progress, ref csvImporter);
            }
        }

        AddScenarioDataToDikes(csvImporter);
        if (importMode == ImportMode.All)
        {
            CorrectSegmentAssignments();
        }
    }

    /// <summary>
    /// Adds the scenario data to dikes.
    /// </summary>
    /// <param name="csvImporter">The CSV importer.</param>
    private void AddScenarioDataToDikes(CsvImporter csvImporter)
    {
        if (csvImporter != null)
        {
            DataSourceManager.CurrentSource = DataSourceSystemType.Csv;
            var combineImportedData = new CombineImportedData
            {
                WaterBoard = waterBoard,
                LocationRecords = csvImporter.LocationRecords,
                ScenarioRecords = csvImporter.ScenariosRecords
            };
            combineImportedData.AddScenarioDataToDikes();
            ImportLogMessages.AddRange(combineImportedData.ErrorMessages);
            Debug.Assert(ReferenceEquals(waterBoard, combineImportedData.WaterBoard));
            waterBoard = combineImportedData.WaterBoard;
        }
    }

    /// <summary>
    /// Adds the CSV data to dikes.
    /// </summary>
    /// <param name="csvImporter">The CSV importer.</param>
    private void AddCsvDataToDikes(CsvImporter csvImporter)
    {
        if (csvImporter != null)
        {
            DataSourceManager.CurrentSource = DataSourceSystemType.Csv;
            var combineImportedData = new CombineImportedData
            {
                WaterBoard = waterBoard,
                LocationRecords = csvImporter.LocationRecords,
                CharacteristicPointsRecords = csvImporter.CharacteristicPointsRecords,
                ScenarioRecords = csvImporter.ScenariosRecords,
                SegmentRecords = csvImporter.SegmentRecords,
                SoilProfilerecords = csvImporter.SoilProfilesRecords,
                SurfaceLineRecords = csvImporter.SurfaceLinesRecords
            };
            combineImportedData.AddCsvDataToDikes();
            ImportLogMessages.AddRange(combineImportedData.ErrorMessages);
            Debug.Assert(ReferenceEquals(waterBoard, combineImportedData.WaterBoard));
            waterBoard = combineImportedData.WaterBoard;
        }
    }

    /// <summary>
    /// When segment shapefile has assigned new segmentIds the segment object has to be re-assigned
    /// </summary>
    private void CorrectSegmentAssignments()
    {
        foreach (Dike dike in waterBoard.Dikes)
        {
            foreach (Location location in dike.Locations)
            {
                location.Segment = waterBoard.Segments.FirstOrDefault(s => s.Name.Equals(location.SegmentId));
            }
        }
    }

    /// <summary>
    /// Check if data definitions already specified
    /// </summary>
    private void ThrowIfNoDataDefinitions()
    {
        if (DataSources == null || !DataSources.Any())
        {
            throw new DataPluginImporterException("No data sources defined");
        }
    }

    /// <summary>
    /// Check if data has been read
    /// </summary>
    private void ThrowIfDataNotRead()
    {
        if (!isAllDataImported)
        {
            throw new DataPluginImporterException("Data has not been imported yet");
        }
    }
}